Unverified Commit efdfd57f authored by Tom Moulard's avatar Tom Moulard Committed by GitHub
Browse files

Merge branch 'master' into master

parents a683ab58 f13ed881

.github/workflows/dockerpublish.yml

+140 −10
Original line number Diff line number Diff line 
name: Docker

name: Tests



on:

  push:

    branches:

      - master

  pull_request:



jobs:

  # Run tests.

  # See also https://docs.docker.com/docker-hub/builds/automated-testing/

  test:

  Config-test:

    runs-on: ubuntu-latest

    steps:

      - uses: actions/checkout@v2

      - name: Run tests

        run: ./test.sh

      - uses: actions/upload-artifact@v1

      - uses: actions/upload-artifact@v2

        if: failure()

        with:

          name: logs

          path: log.log
 
 No newline at end of file
 
          name: test-artifacts

          path: |

            log.log

            *.patch

  Health-checks-nginx:

    runs-on: ubuntu-latest

    env:

      SERVICE: nginx

    steps:

      - uses: actions/checkout@v2

      - name: Caching

        uses: actions/cache@v2

        with:

          path: /var/lib/docker/

          key: ${{ runner.os }}-health-${{ github.job }}

      - name: Build the docker-compose stack

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml up -d

      - name: Waiting for service startup (10s)

        run: sleep 10s

      - name: Check running containers

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps

      - name: Check health

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps | grep "Up (healthy)"

  Health-checks-blog:

    runs-on: ubuntu-latest

    env:

      SERVICE: blog

    steps:

      - uses: actions/checkout@v2

      - name: Caching

        uses: actions/cache@v2

        with:

          path: /var/lib/docker/

          key: ${{ runner.os }}-health-${{ github.job }}

      - name: Build the docker-compose stack

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml up -d

      - name: Waiting for service startup (10s)

        run: sleep 10s

      - name: Check running containers

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps

      - name: Check health

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps | grep "Up (healthy)"

  Health-checks-peertube:

    runs-on: ubuntu-latest

    env:

      SERVICE: peertube

    steps:

      - uses: actions/checkout@v2

      - name: Caching

        uses: actions/cache@v2

        with:

          path: /var/lib/docker/

          key: ${{ runner.os }}-health-${{ github.job }}

      - name: Build the docker-compose stack

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml up -d

      - name: Waiting for service startup (10s)

        run: sleep 10s

      - name: Check running containers

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps

      - name: Check health

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps | grep "Up (healthy)"

  Health-checks-transmission:

    runs-on: ubuntu-latest

    env:

      SERVICE: transmission

    steps:

      - uses: actions/checkout@v2

      - name: Caching

        uses: actions/cache@v2

        with:

          path: /var/lib/docker/

          key: ${{ runner.os }}-health-${{ github.job }}

      - name: Build the docker-compose stack

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml up -d

      - name: Waiting for service startup (10s)

        run: sleep 10s

      - name: Check running containers

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps

      - name: Check health

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps | grep "Up (healthy)"

  Health-checks-nextcloud:

    runs-on: ubuntu-latest

    env:

      SERVICE: nextcloud

    steps:

      - uses: actions/checkout@v2

      - name: Caching

        uses: actions/cache@v2

        with:

          path: /var/lib/docker/

          key: ${{ runner.os }}-health-${{ github.job }}

      - name: Build the docker-compose stack

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml up -d

      - name: Waiting for service startup (10s)

        run: sleep 10s

      - name: Check running containers

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps

      - name: Check health

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps | grep "Up (healthy)"

  Health-checks-jackett:

    runs-on: ubuntu-latest

    env:

      SERVICE: jackett

    steps:

      - uses: actions/checkout@v2

      - name: Caching

        uses: actions/cache@v2

        with:

          path: /var/lib/docker/

          key: ${{ runner.os }}-health-${{ github.job }}

      - name: Build the docker-compose stack

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml up -d

      - name: Waiting for service startup (10s)

        run: sleep 10s

      - name: Check running containers

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps

      - name: Check health

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps | grep "Up (healthy)"

  Health-checks-codimd:

    runs-on: ubuntu-latest

    env:

      SERVICE: codimd

    steps:

      - uses: actions/checkout@v2

      - name: Caching

        uses: actions/cache@v2

        with:

          path: /var/lib/docker/

          key: ${{ runner.os }}-health-${{ github.job }}

      - name: Build the docker-compose stack

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml up -d

      - name: Waiting for service startup (10s)

        run: sleep 10s

      - name: Check running containers

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps

      - name: Check health

        run: docker-compose -f docker-compose.yml -f ${SERVICE}/docker-compose.${SERVICE}.yml ps | grep "Up (healthy)"

.gitignore

+2 −0
Original line number Diff line number Diff line
@@ -4,3 +4,5 @@ blog/nginx/conf/www 
gitlab/logs

portainer/data

.env

*.patch

*.swp

README.md

+32 −123
Original line number Diff line number Diff line
@@ -12,88 +12,39 @@ SITE=tom.moulard.org docker-compose up -d 


Now you have my own server configuration.



To be a little more consistent with the management, you can use a `.env` file and do:

To be a little more consistent with the management, you can use a `.env` file

and do:

```bash

cp .env.default .env

```



and edit the file to use the correct site url.

And edit the file to use the correct site URL.



The `docker-compose` function gather all docker-compose files in order to have the whole configuration in one place (`docker-compose config`).

The `docker-compose` function gather all docker-compose files in order to have

the whole configuration in one place (see `docker-compose config`).



### Tear down

```bash

docker-compose down

```



## TODO

### New ideas

 - [X] traefik

 - [X] gitlab

    - [X] CI/CD worker(s)

 - [X] nextcloud

 - [X] nginx

 - [X] weechat

 - [X] transmission

 - [X] vpn

 - [X] jupyter

 - [ ] readthedoc / [DokuWiki](https://hub.docker.com/r/mprasil/dokuwiki)

 - [X] pastebin

 - [ ] image / screenshot hosting

 - [ ] [hackmd](https://github.com/hackmdio/docker-hackmd) [main repo](https://github.com/hackmdio/codimd)

 - [X] [prometheus](https://www.brianchristner.io/how-to-monitor-traefik-reverse-proxy-with-prometheus/) / [EFK](https://docs.fluentd.org/v0.12/articles/docker-logging-efk-compose) / [filebeat](https://medium.com/the-sysadmin/visualize-traefik-logs-in-kibana-c53fb2aac070) / ELK

 - [ ] proxy

 - [ ] [RSS agregator server](https://www.freshrss.org/)

 - [ ] url shortener

 - [ ] factorio server

 - [ ] news group server

 - [ ] vlc server

 - [X] blog using [hugo](http://gohugo.io)

 - [ ] MOOC

 - [ ] Latex online editor

 - [X] [Bazarr](https://hub.docker.com/r/linuxserver/bazarr) (subs), [lidarr](https://hub.docker.com/r/linuxserver/lidarr) (music), [sonarr](https://hub.docker.com/r/linuxserver/sonarr) (shows), [jackett](https://hub.docker.com/r/linuxserver/jackett) (interface)

 - [X] Bitwarden



[more](https://github.com/Kickball/awesome-selfhosted)



### List

 - [ ] which database ? maria / mysql / mongo / postgres

    - [ ] gitlab postgresSQL / MySQL - MariaDB

    - [ ] nextcloud postgresSQL / MySQL - MariaDB / Oracle

 - [X] nginx.conf

 - [ ] create a git repository auto in gitlab for // FIXME

 - [ ] Create a Dockerfile for a mail server

 - [X] reverse proxy with ssl

 - [ ] multi files configuration

 - [ ] Testing

    - [X] traefik

    - [X] gitlab

    - [X] nextcloud

    - [X] nginx

    - [ ] weechat

    - [X] transmission

    - [X] vpn

    - [X] jupyter

    - [X] pastebin

    - [X] ELK

    - [ ] sharelatex / Overleaf

    - [X] blog



### Configuration files

 - [ ] have default configuration files

    - [X] traefik

    - [X] gitlab

    - [X] gitlab runner

    - [X] transmission

    - [ ] pastebin

    - [ ] nextcloud

    - [X] nginx

    - [X] ELK (WIP)

### Services list

There **should** be only one service by folder:

For example, le folder `traefik/` contains all the necessary configuration to

run the `traefik` service.



Thus each folder represent an available service.



Feel free to do a Pull Request to add your ideas.



[more ideas](https://github.com/awesome-selfhosted/awesome-selfhosted)



## Configuration

Don't forget to change db passwords. (migth not be needed since they are beyond

the reverse proxy).

Fill vpn secrets(if none provided, they are generated directly).

Don't forget to change:



 - db passwords (might not be needed since they are beyond the reverse proxy)

 - VPN secrets (if none provided, they are generated directly).



Configuration files are: `docker-compose.yml`, `nginx.conf`



To set the password:
@@ -106,65 +57,18 @@ You can add a new set of credentials by editing the .env file like 
USERS=toto:pass,tata:pass, ...

```



### Scalling up

```bash

docker-compose scale nginx=2

```

### For local developments

Edit the file `/etc/hosts` to provide the reverse proxy with good URLs.



### Adress table

| Status | Address | port(s)|

|:--:|--|--|

| [X] | traefik.${SITE} | 80, 443 (redirect 80 to 443) |

| [X] | gitlab.${SITE} | 22, 80, 443 |

| [X] | cloud.${SITE} | 80, 443 |

| [X] | ${SITE} | 80, 443 |

| [ ] | mail.${SITE} | 25(recv mail), 465(ssl), 587(TLS), 143(IMAP), 993(IMAP), 110(POP3), 995(POP3) |

| [X] | torrent.${SITE} | 80, 443 (redirect 80 to 443) |

| [X] | vpn.${SITE} | 500, 4500 |

| [X] | jupiter.${SITE} | 80, 443 (redirect 80 to 443) |

| [X] | paste.${SITE} | 80, 443 (redirect 80 to 443) |

| [X] | video.${SITE} | 80, 443 (redirect 80 to 443) |

| [ ] | irc.${SITE} | ?? |



### Gitlab runner

#### Get the Registration Token

Find your runner registration token ($REGISTRATION_TOKEN) at `http://GITLAB_HOST/$PROJECT_GROUP/$PROJECT_NAME/settings/ci_cd`.



There is **two** way to register the runner:



##### Register via config file

Register the Registration Token to have a Runner Token

For example, adding this in your `/etc/hosts` will allow to run and debug the

Traefik service locally:

```bash

curl -X POST 'http://gitlab.${SITE}/api/v4/runners' --form 'token=$REGISTRATION_TOKEN' --form 'description=The Best Runner'

127.0.0.1   traefik.moulard.org

```



###### Change runner configuration

Now change the token in the [configuration file](gitlab/runner/config.toml).

```toml

[[runners]]

    token = "XXXXXXXXXXXXXXXXXXXX"

```

and run the runner

```bash

docker-compose up -d runner

```



##### Register via CLI

Steps:

 - up the runner `docker-compose up -d runner`

 - register the runner

### Scaling up

```bash

docker-compose exec runner gitlab-runner register \

    --non-interactive \

    --executor "docker" \

    --docker-image alpine:latest \

    --url "http://gitlab/" \

    --registration-token "$REGISTRATION_TOKEN" \

    --description "The Best Runner" \

    --tag-list "docker,aws" \

    --run-untagged="true" \

    --locked="false" \

    --access-level="not_protected"

docker-compose scale nginx=2

```



# Authors
@@ -173,3 +77,8 @@ Main author: 


Gitlab helper:

 - [michel_k](mailto:thomas.michelot@epita.fr)



Discord MusicBot:

 - [huvell_m](mailto:martin.huvelle@epita.fr),

see PR [#6](https://github.com/tomMoulard/make-my-server/pull/6)

blog/docker-compose.blog.yml

+5 −0
Original line number Diff line number Diff line
@@ -11,6 +11,11 @@ services: 
    networks:

      - 'srv'

    restart: always

    healthcheck:

      test: ['CMD', 'curl', '0.0.0.0:80']

      interval: 10s

      timeout: 10s

      retries: 5

    labels:

      - 'traefik.enable=true'

      - 'traefik.frontend.rule=Host:blog.${SITE}'

codimd/docker-compose.codimd.yml

+17 −2
Original line number Diff line number Diff line 
version: '2'



networks:

    codi-internal:



services:

  codimd:

    image: nabo.codimd.dev/hackmdio/hackmd:2.1.0

    image: nabo.codimd.dev/hackmdio/hackmd:2.2.0

    environment:

      - 'CMD_DB_URL=postgres://codimd:mypwd@codimd-db/codimd'

      - 'CMD_USECDN=false'
@@ -14,11 +17,18 @@ services: 
      - './codimd/data:/home/hackmd/app/public/uploads'

    networks:

      - 'srv'

      - 'codi-internal'

    restart: always

    healthcheck:

      test: ['CMD', 'wget', '0.0.0.0:3000', '|', 'grep', '200']

      interval: 10s

      timeout: 10s

      retries: 5

    labels:

      - 'traefik.enable=true'

      - 'traefik.frontend.rule=Host:codimd.${SITE}'

      - 'traefik.port=3000'



  codimd-db:

    image: postgres:11.6-alpine

    environment:
@@ -28,7 +38,12 @@ services: 
    volumes:

      - './codimd/db:/var/lib/postgresql/data'

    networks:

      - 'srv'

      - 'codi-internal'

    restart: always

    healthcheck:

      test: ['CMD', 'pg_isready', '-U', 'codimd']

      interval: 10s

      timeout: 10s

      retries: 5

    labels:

      - 'traefik.enable=false'