commit d85a0d60b4760294cec46ac7f541b1efab0ebe5d Author: Sven Velt Date: Tue Dec 1 23:20:33 2020 +0100 Initial commit (submodules pending) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..75540da --- /dev/null +++ b/.gitignore @@ -0,0 +1,155 @@ +# ---> Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# ---> Vim +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +# ---> Ansible +*.retry + diff --git a/webcluster-base/defaults/main.yml b/webcluster-base/defaults/main.yml new file mode 100644 index 0000000..c38fd12 --- /dev/null +++ b/webcluster-base/defaults/main.yml @@ -0,0 +1,8 @@ +--- +ntp_servers: + - 0.de.pool.ntp.org + - 1.de.pool.ntp.org + - 2.de.pool.ntp.org + +ntp_service_name: ntp + diff --git a/webcluster-base/meta/main.yml b/webcluster-base/meta/main.yml new file mode 100644 index 0000000..56df01f --- /dev/null +++ b/webcluster-base/meta/main.yml @@ -0,0 +1,5 @@ +--- +dependencies: + - role: ntp + when: ansible_distribution != "Alpine" + diff --git a/webcluster-db.yml b/webcluster-db.yml new file mode 100644 index 0000000..60c9b72 --- /dev/null +++ b/webcluster-db.yml @@ -0,0 +1,11 @@ +--- +- hosts: db + tasks: [] + +- hosts: db + serial: 1 + max_fail_percentage: 20 + roles: + - mariadb-galera + - webcluster-db + diff --git a/webcluster-db/README.md b/webcluster-db/README.md new file mode 100644 index 0000000..3943320 --- /dev/null +++ b/webcluster-db/README.md @@ -0,0 +1,39 @@ +webcluster-db +============= + +Prepare database for web cluster + +Requirements +------------ + +At least one MariaDB server (or cluster) + +Role Variables +-------------- + +- None + + +Dependencies +------------ + +- None + +Example Playbook +---------------- + + - hosts: webcluster-db + roles: + - { role: webcluster-db } + +License +------- + +GPL-2.0-or-later + +Author Information +------------------ + +- Sven Velt +- https://git.velt.biz/velt.biz/ + diff --git a/webcluster-db/defaults/main.yml b/webcluster-db/defaults/main.yml new file mode 100644 index 0000000..26d5e2f --- /dev/null +++ b/webcluster-db/defaults/main.yml @@ -0,0 +1,10 @@ +--- +db_packages: + - mariadb-server + - python3-mysqldb + +db_user: webappuser +db_password: webapppw + +db_name: webapp + diff --git a/webcluster-db/meta/main.yml b/webcluster-db/meta/main.yml new file mode 100644 index 0000000..d98ee75 --- /dev/null +++ b/webcluster-db/meta/main.yml @@ -0,0 +1,21 @@ +galaxy_info: + author: Sven Velt + description: Prepare database for web cluster + company: velt.biz + + issue_tracker_url: https://git.velt.biz/velt.biz/ansible_role_webcluster-db/issues/ + + license: GPL-2.0-or-later + + min_ansible_version: 2.9 + + platforms: + - name: Ubuntu + versions: + - focal + + galaxy_tags: + - database + - db + - mariadb + diff --git a/webcluster-db/tasks/main.yml b/webcluster-db/tasks/main.yml new file mode 100644 index 0000000..8f8e72e --- /dev/null +++ b/webcluster-db/tasks/main.yml @@ -0,0 +1,25 @@ +--- +- name: Install software + package: + name: "{{ item }}" + state: present + with_items: "{{ db_packages }}" + + +- name: MariaDB create DB + mysql_db: + name: "{{ db_name }}" + state: present + run_once: True + + +- name: MariaDB create user + mysql_user: + name: "{{ db_user }}" + password: "{{ db_password }}" + priv: '{{ db_name }}.*:ALL' + host: '%' + state: present + run_once: True + + diff --git a/webcluster-haproxy/README.md b/webcluster-haproxy/README.md new file mode 100644 index 0000000..f57416a --- /dev/null +++ b/webcluster-haproxy/README.md @@ -0,0 +1,38 @@ +webcluster-haproxy +================== + +Setup haproxy for webcluster + +Requirements +------------ + +- Inventory group "worker" with worker nodes + +Role Variables +-------------- + +- haproxy_sslcert_path: Path to key/cert file + +Dependencies +------------ + +- None + +Example Playbook +---------------- + + - hosts: webcluster-lb + roles: + - { role: webcluster-haproxy } + +License +------- + +GPL-2.0-or-later + +Author Information +------------------ + +- Sven Velt +- https://git.velt.biz/velt.biz/ + diff --git a/webcluster-haproxy/defaults/main.yml b/webcluster-haproxy/defaults/main.yml new file mode 100644 index 0000000..f9ef36e --- /dev/null +++ b/webcluster-haproxy/defaults/main.yml @@ -0,0 +1,8 @@ +--- +haproxy_packages: + - haproxy + - socat + - rsyslog + +haproxy_path_socket: /var/lib/haproxy/admin.sock + diff --git a/webcluster-haproxy/handlers/main.yml b/webcluster-haproxy/handlers/main.yml new file mode 100644 index 0000000..b8068c9 --- /dev/null +++ b/webcluster-haproxy/handlers/main.yml @@ -0,0 +1,13 @@ +--- +- name: Restart HAProxy + service: + name: haproxy + state: restarted + + +- name: Restart RSyslog + service: + name: rsyslog + state: restarted + + diff --git a/webcluster-haproxy/meta/main.yml b/webcluster-haproxy/meta/main.yml new file mode 100644 index 0000000..c1391c4 --- /dev/null +++ b/webcluster-haproxy/meta/main.yml @@ -0,0 +1,22 @@ +galaxy_info: + author: Sven Velt + description: Setup haproxy for webcluster + company: velt.biz + + issue_tracker_url: https://git.velt.biz/velt.biz/ansible_role_webcluster-haproxy/issues/ + + license: GPL-2.0-or-later + + min_ansible_version: 2.9 + + platforms: + - name: Alpine + versions: + - 3.11 + - 3.12 + + galaxy_tags: + - haproxy + - reverseproxy + - web + diff --git a/webcluster-haproxy/tasks/main.yml b/webcluster-haproxy/tasks/main.yml new file mode 100644 index 0000000..491e9a0 --- /dev/null +++ b/webcluster-haproxy/tasks/main.yml @@ -0,0 +1,66 @@ +--- +- name: Install software + package: + name: "{{ item }}" + state: latest + with_items: "{{ haproxy_packages }}" + notify: Restart RSyslog + + +- name: HAProxy create config + template: + dest: "/etc/haproxy/haproxy.cfg" + src: "haproxy.cfg.j2" + mode: 0644 + backup: yes + notify: + - Restart HAProxy + - Restart RSyslog + + +- name: Create Rsyslog config dir + file: + dest: /etc/rsyslog.d/ + state: directory + owner: root + group: root + mode: 0644 + + +- name: Rsyslog create config + template: + dest: "/etc/rsyslog.d/haproxy.conf" + src: "rsyslog.conf.j2" + mode: 0644 + backup: yes + notify: + - Restart RSyslog + + +- name: Create jail /dev directory + file: + path: /var/lib/haproxy/dev/ + state: directory + owner: root + group: root + mode: 0755 + + +- name: Copy SSL certificate + copy: + src: "{{ haproxy_sslcert_src }}" + dest: "{{ haproxy_sslcert_path }}" + owner: root + group: root + mode: 0600 + backup: yes + when: haproxy_sslcert_src|default("") != "" + + +- name: HAProxy enable service + service: + name: haproxy + enabled: yes + state: started + + diff --git a/webcluster-haproxy/templates/haproxy.cfg.j2 b/webcluster-haproxy/templates/haproxy.cfg.j2 new file mode 100644 index 0000000..08dd39e --- /dev/null +++ b/webcluster-haproxy/templates/haproxy.cfg.j2 @@ -0,0 +1,101 @@ +# ----- global configuration ----- # +global + log /dev/log local0 + + chroot /var/lib/haproxy + pidfile /var/run/haproxy.pid + + user haproxy + group haproxy + daemon + + # turn on stats unix socket + stats socket {{ haproxy_path_socket }} mode 660 level admin + stats timeout 30s + + # Default SSL material locations + ca-base /etc/ssl/certs + crt-base /etc/ssl/private + + # Default ciphers to use on SSL-enabled listening sockets. + # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ + ssl-default-bind-options no-sslv3 + ssl-default-bind-ciphers ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS:!AESCCM + + ssl-default-server-options no-sslv3 + ssl-default-server-ciphers ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS:!AESCCM + + tune.ssl.default-dh-param 2048 + + +# ----- default configuration options ----- # +defaults + mode http + balance roundrobin + log global + option httplog + option dontlognull + option http-server-close + option forwardfor except 127.0.0.0/8 + option redispatch + option httpchk HEAD /haproxy.check HTTP/1.1\r\nHost:localhost + + maxconn 3000 + retries 3 + + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 10s + timeout check 10s + + # errorfile 400 /etc/haproxy/errors/400.http + # errorfile 403 /etc/haproxy/errors/403.http + # errorfile 408 /etc/haproxy/errors/408.http + # errorfile 500 /etc/haproxy/errors/500.http + # errorfile 502 /etc/haproxy/errors/502.http + # errorfile 503 /etc/haproxy/errors/503.http + # errorfile 504 /etc/haproxy/errors/504.http + + +# ----- statistics frontend ----- # +listen stats + bind :::1936 v4v6 + stats enable + stats uri / + stats refresh 30s + #stats hide-version + stats auth admin:admin + + +# ----- frontends ----- # +frontend http + bind :::80 v4v6 +{% if haproxy_sslcert_src|default(False) %} bind :::443 v4v6 ssl crt /etc/haproxy/ssl.pem +{% endif %} + mode http + + acl url_static path_beg -i /static /images /javascript /stylesheets + acl url_static path_end -i .jpg .jpeg .gif .png .html .htm .css .js + + use_backend static if url_static + default_backend dynamic + + +# ----- backends ----- # +backend static +{% for host in groups['worker'] %} + server {{ host }} {{ hostvars[host]['ansible_default_ipv4'].address }}:80 # check +{% endfor %} + + +backend dynamic + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { ssl_fc } + +{% for host in groups['worker'] %} + server {{ host }} {{ hostvars[host]['ansible_default_ipv4'].address }}:80 check inter 2s fastinter 1s fall 3 rise 2 +{% endfor %} + diff --git a/webcluster-haproxy/templates/rsyslog.conf.j2 b/webcluster-haproxy/templates/rsyslog.conf.j2 new file mode 100644 index 0000000..b49b290 --- /dev/null +++ b/webcluster-haproxy/templates/rsyslog.conf.j2 @@ -0,0 +1,12 @@ +# Doppelt hält besser :-/ +module(load="imuxsock") + +input(type="imuxsock" + Socket="/var/lib/haproxy/dev/log" +) + +:programname, startswith, "haproxy" { + /var/log/haproxy.log + stop +} + diff --git a/webcluster-lb.yml b/webcluster-lb.yml new file mode 100644 index 0000000..ec1876f --- /dev/null +++ b/webcluster-lb.yml @@ -0,0 +1,9 @@ +--- +- hosts: worker + tasks: [] + +- hosts: lb + roles: + - keepalived-simple + - webcluster-haproxy + diff --git a/webcluster-site.yml b/webcluster-site.yml new file mode 100644 index 0000000..f4bd562 --- /dev/null +++ b/webcluster-site.yml @@ -0,0 +1,29 @@ +--- +- hosts: + - lb + - worker + - db + roles: + - webcluster-base + +- hosts: lb + serial: "25%" + max_fail_percentage: 40 + roles: + - keepalived-simple + - webcluster-haproxy + +- hosts: worker + serial: 2 + max_fail_percentage: 20 + roles: + - webcluster-worker + - apache2-backend-mod_remoteip + +- hosts: db + serial: 1 + max_fail_percentage: 10 + roles: + - mariadb-galera + - webcluster-db + diff --git a/webcluster-worker.yml b/webcluster-worker.yml new file mode 100644 index 0000000..390c65c --- /dev/null +++ b/webcluster-worker.yml @@ -0,0 +1,8 @@ +--- +- hosts: worker + serial: 2 + max_fail_percentage: 20 + roles: + - webcluster-worker + - apache2-backend-mod_remoteip + diff --git a/webcluster-worker/README.md b/webcluster-worker/README.md new file mode 100644 index 0000000..f59973f --- /dev/null +++ b/webcluster-worker/README.md @@ -0,0 +1,39 @@ +webcluster-worker +================= + +Setup worker for web cluster + +Requirements +------------ + +None + +Role Variables +-------------- + +- None + + +Dependencies +------------ + +- None + +Example Playbook +---------------- + + - hosts: webcluster-worker + roles: + - { role: webcluster-worker } + +License +------- + +GPL-2.0-or-later + +Author Information +------------------ + +- Sven Velt +- https://git.velt.biz/velt.biz/ + diff --git a/webcluster-worker/defaults/main.yml b/webcluster-worker/defaults/main.yml new file mode 100644 index 0000000..5793ee4 --- /dev/null +++ b/webcluster-worker/defaults/main.yml @@ -0,0 +1,27 @@ +--- +worker_packages: + - apache2 + - php-fpm + - php-mysql + +worker_a2mods: + - mpm_event + - proxy + - proxy_fcgi + - rewrite + +worker_a2dissites: + - 000-default.conf + - default-ssl.conf + +worker_a2ensites: + - worker.conf + +worker_a2docroots: + - var/www/worker + +worker_phpfpmpools: + - worker.conf + +worker_phpversion: 7.4 + diff --git a/webcluster-worker/handlers/main.yml b/webcluster-worker/handlers/main.yml new file mode 100644 index 0000000..9303854 --- /dev/null +++ b/webcluster-worker/handlers/main.yml @@ -0,0 +1,13 @@ +--- +- name: Restart Apache2 + service: + name: apache2 + state: restarted + + +- name: Restart PHP-FPM + service: + name: "php{{ worker_phpversion }}-fpm" + state: restarted + + diff --git a/webcluster-worker/meta/main.yml b/webcluster-worker/meta/main.yml new file mode 100644 index 0000000..4b1801a --- /dev/null +++ b/webcluster-worker/meta/main.yml @@ -0,0 +1,22 @@ +galaxy_info: + author: Sven Velt + description: Setup worker for web cluster + company: velt.biz + + issue_tracker_url: https://git.velt.biz/velt.biz/ansible_role_webcluster-worker/issues/ + + license: GPL-2.0-or-later + + min_ansible_version: 2.9 + + platforms: + - name: Ubuntu + versions: + - focal + + galaxy_tags: + - apache + - php + - web + - webserver + diff --git a/webcluster-worker/tasks/main.yml b/webcluster-worker/tasks/main.yml new file mode 100644 index 0000000..ef2b8a6 --- /dev/null +++ b/webcluster-worker/tasks/main.yml @@ -0,0 +1,103 @@ +--- +- name: Install software + package: + name: "{{ worker_packages }}" + state: latest + + +- name: Apache2 enable modules + apache2_module: + name: "{{ item }}" + state: present + with_items: "{{ worker_a2mods }}" + notify: Restart Apache2 + + +- name: Apache2 disable sites + file: + path: "/etc/apache2/sites-enabled/{{ item }}" + state: absent + with_items: "{{ worker_a2dissites }}" + notify: Restart Apache2 + + +- name: Apache2 create vhosts + template: + dest: "/etc/apache2/sites-available/{{ item }}" + src: "etc/apache2/sites-available/{{ item }}.j2" + mode: 0644 + backup: yes + with_items: "{{ worker_a2ensites }}" + notify: Restart Apache2 + + +- name: Apache2 enable sites + file: + path: "/etc/apache2/sites-enabled/{{ item }}" + src: "/etc/apache2/sites-available/{{ item }}" + state: link + force: yes + with_items: "{{ worker_a2ensites }}" + notify: Restart Apache2 + + +- name: Apache2 create DocRoots + file: + path: "/{{ item }}" + state: directory + mode: 0755 + with_items: "{{ worker_a2docroots }}" + notify: Restart Apache2 + + +- name: PHP Install pools + template: + dest: "/etc/php/{{ worker_phpversion }}/fpm/pool.d/{{ item }}" + src: "etc/php-fpm-pool.d/{{ item }}.j2" + mode: 0644 + backup: yes + with_items: "{{ worker_phpfpmpools }}" + notify: Restart PHP-FPM + + +- name: Disable worker in load balancers + haproxy: + socket: /var/lib/haproxy/admin.sock + backend: dynamic + host: "{{ inventory_hostname }}" + state: disabled + delegate_to: "{{ item }}" + with_items: "{{ groups.lb }}" + + +- name: Apache2 copy websites + copy: + dest: "/{{ item }}/" + src: "{{ item }}/" + backup: yes + with_items: "{{ worker_a2docroots }}" + + +- name: Apache2 template dummy index.html + template: + dest: "/{{ item }}/index.html" + src: "{{ item }}/index.html.j2" + mode: 0644 + backup: yes + with_items: "{{ worker_a2docroots }}" + + +- name: Sleep 30 seconds... + pause: seconds=30 + + +- name: Enable worker in load balancers + haproxy: + socket: /var/lib/haproxy/admin.sock + backend: dynamic + host: "{{ inventory_hostname }}" + state: enabled + delegate_to: "{{ item }}" + with_items: "{{ groups.lb }}" + + diff --git a/webcluster-worker/templates/etc/apache2/sites-available/worker.conf.j2 b/webcluster-worker/templates/etc/apache2/sites-available/worker.conf.j2 new file mode 100644 index 0000000..7a7df93 --- /dev/null +++ b/webcluster-worker/templates/etc/apache2/sites-available/worker.conf.j2 @@ -0,0 +1,13 @@ + + ServerName {{ inventory_hostname }} + DocumentRoot /var/www/worker + + ProxyPassMatch ^/(.*\.php(/.*)?)$ fcgi://127.0.0.1:4001/var/www/worker/$1 + + DirectoryIndex index.php index.xhtml index.html + + SetEnvIf Request_URI "^/haproxy\.check$" dontlog + CustomLog ${APACHE_LOG_DIR}/worker_{{ inventory_hostname }}.access.log remoteip_combined env=!dontlog + + ErrorLog ${APACHE_LOG_DIR}/worker_{{ inventory_hostname }}.error.log + diff --git a/webcluster-worker/templates/etc/php-fpm-pool.d/worker.conf.j2 b/webcluster-worker/templates/etc/php-fpm-pool.d/worker.conf.j2 new file mode 100644 index 0000000..2b96120 --- /dev/null +++ b/webcluster-worker/templates/etc/php-fpm-pool.d/worker.conf.j2 @@ -0,0 +1,12 @@ +[worker] +user = www-data +group = www-data +listen = 127.0.0.1:4001 +listen.owner = www-data +listen.group = www-data +pm = dynamic +pm.max_children = 5 +pm.start_servers = 3 +pm.min_spare_servers = 1 +pm.max_spare_servers = 3 +