feat: add mysql/mysql_exporter/fluentd

master
Dave S. 2 years ago
parent f7b202ad7f
commit 7e2f7e5749
  1. 7
      README.md
  2. 1
      all.yml
  3. 6
      ansible.yml
  4. 9
      group_vars/all.yml
  5. 5
      group_vars/containers.yml
  6. 78
      hosts
  7. 25
      roles/ansible/tasks/main.yml
  8. 17
      roles/container/tasks/main.yml
  9. 118
      roles/fluentd/tasks/prepare_build.yml
  10. 41
      roles/fluentd/templates/fluentd.j2
  11. 3
      roles/hv/defaults/main.yml
  12. 37
      roles/hv/tasks/install_docker.yml
  13. 12
      roles/mysql/tasks/prepare_build.yml
  14. 37
      roles/mysql_exporter/tasks/main.yml
  15. 7
      roles/mysql_exporter/tasks/prepare_build.yml
  16. 9
      roles/nginx/templates/nginx.j2
  17. 2
      roles/nginx_exporter/tasks/prepare_build.yml
  18. 44
      tasks/add_docker_repo.yml
  19. 13
      tasks/init_mysql.yml
  20. 14
      templates/fluentd.Dockerfile.j2

@ -2,6 +2,7 @@
## Пререквизиты ## Пререквизиты
- ВМ или контейнер с Debian 11, который будет работать в качестве Docker-хоста (Docker ставить не нужно) - ВМ или контейнер с Debian 11, который будет работать в качестве Docker-хоста (Docker ставить не нужно)
- Ansible-контроллер под Alpine или Debian
- настроено SSH-подключение под учеткой root (для Ansible) - настроено SSH-подключение под учеткой root (для Ansible)
- Python 3 (для Ansible) - Python 3 (для Ansible)
- две записи в файле hosts: - две записи в файле hosts:
@ -34,12 +35,12 @@ docker_network_iprange: 10.90.0.0/16
- [x] apache2 - [x] apache2
- [x] node_exporter - [x] node_exporter
- [x] nginx_exporter - [x] nginx_exporter
- [ ] mysql_exporter - [x] mysql_exporter
- [x] cadvisor - [x] cadvisor
- [ ] prometheus - [ ] prometheus
- [ ] grafana - [ ] grafana
- [ ] fluentd - [x] fluentd
- [ ] mysql - [x] mysql
## Исходные данные ## Исходные данные

@ -1,2 +1,3 @@
- import_playbook: ansible.yml
- import_playbook: hv.yml - import_playbook: hv.yml
- import_playbook: containers.yml - import_playbook: containers.yml

@ -0,0 +1,6 @@
- hosts: localhost
gather_facts: no
serial: 1
strategy: linear
roles:
- ansible

@ -3,3 +3,12 @@ container_logs_dir: /opt/logs
container_root_build_dir: /opt/build container_root_build_dir: /opt/build
docker_remote_port: 5011 docker_remote_port: 5011
fluentd_database_name: nginx
fluentd_database_user: fluentd
fluentd_database_pass: 4CDVyJQPGsrzmy3U3XgdYEKGGxoRBquy
fluentd_database_table: nginx_logs
gpg_keyrings_dir: /etc/apt/trusted.gpg.d
docker_repo_url: https://download.docker.com/linux/debian

@ -1,4 +1,5 @@
ansible_connection: docker
hypervisor_hostname: hv hypervisor_hostname: hv
container_default_config: ansible_connection: docker
ansible_user: root
ansible_docker_extra_args: "-H tcp://{{ hostvars[hypervisor_hostname]['ansible_host'] ~ ':' ~ docker_remote_port }}"

78
hosts

@ -10,8 +10,7 @@ all:
containers: containers:
hosts: hosts:
nginx: nginx:
ansible_host: 10.90.0.2 container_address: 10.90.0.2
ansible_password: k3PpKHF52zgUaowB73V6ggrnMUDUcMeS
container_config: container_config:
image: nginx image: nginx
custom_image: yes custom_image: yes
@ -20,29 +19,42 @@ all:
error_log: /var/log/nginx/error.log info error_log: /var/log/nginx/error.log info
load_module: /usr/local/nginx/modules/ngx_http_geoip2_module.so load_module: /usr/local/nginx/modules/ngx_http_geoip2_module.so
http: http:
log_format: "custom escape=json '\"$time_iso8601\" \"$request_time\" \"$upstream_response_time\" \ log_format: "custom escape=json '{\
\"$remote_addr\" \"$remote_user\" \"$time_local\" \"$request\" \"$status\" \ \"time_iso8601\": \"$time_iso8601\", \
\"$body_bytes_sent\" \"$http_referer\" \"$http_user_agent\" \"$geoip_country_code\"'" \"request_time\": \"$request_time\", \
\"upstream_response_time\": \"$upstream_response_time\", \
\"remote_addr\": \"$remote_addr\", \
\"remote_user\": \"$remote_user\", \
\"time_local\": \"$time_local\", \
\"request\": \"$request\", \
\"status\": \"$status\", \
\"body_bytes_sent\": \"$body_bytes_sent\", \
\"http_referer\": \"$http_referer\", \
\"http_user_agent\": \"$http_user_agent\", \
\"geoip_country_code\": \"$geoip_country_code\"}'"
http_post:
access_log: "syslog:server=10.90.0.10:11514,tag=nginx_access custom"
fluentd_source:
_type: syslog
port: 11514
tag: nginx.access
apache1: apache1:
ansible_host: 10.90.0.3 container_address: 10.90.0.3
ansible_password: qUTuyFHzCFGVswGYEHv5MU2JzQGt9Tx7
host_role: apache host_role: apache
container_config: container_config:
image: php:8-apache image: php:8-apache
mounts: /opt/www/html:/var/www/html mounts: /opt/www/html:/var/www/html
apache2: apache2:
ansible_host: 10.90.0.4 container_address: 10.90.0.4
ansible_password: F2a4v4LoQ5U6rwAgsrSt68SJwmHGARuP
host_role: apache host_role: apache
container_config: container_config:
image: php:8-apache image: php:8-apache
mounts: /opt/www/html:/var/www/html mounts: /opt/www/html:/var/www/html
cadvisor: cadvisor:
ansible_host: 10.90.0.5 container_address: 10.90.0.5
ansible_password: dJpxV9gri438UbC82bYXCp4BeAGHyoZn
container_config: container_config:
image: gcr.io/cadvisor/cadvisor:latest image: gcr.io/cadvisor/cadvisor:latest
mounts: mounts:
@ -56,8 +68,7 @@ all:
- /dev/kmsg - /dev/kmsg
node_exporter: node_exporter:
ansible_host: 10.90.0.6 container_address: 10.90.0.6
ansible_password: QnSKHZ5e82YnCzbS75PMHXZqhwpFbzcu
container_config: container_config:
image: prom/node-exporter:latest image: prom/node-exporter:latest
extra_mounts: extra_mounts:
@ -65,23 +76,36 @@ all:
command: '--path.rootfs=/host' command: '--path.rootfs=/host'
nginx_exporter: nginx_exporter:
ansible_host: 10.90.0.7 container_address: 10.90.0.7
ansible_password: rkeE2vHp2jmLjsyz4r9ASe63GM5t6FbA
container_config: container_config:
image: nginx/nginx-prometheus-exporter image: nginx/nginx-prometheus-exporter
mysql:
container_address: 10.90.0.8
container_config:
image: yobasystems/alpine-mariadb:latest
mounts: /opt/mariadb:/var/lib/mysql
command: '--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci'
mysql_root_password: eJdr5qyXoH2LCna3FaL9w3pAzpxGdWSE
mysql_exporter: mysql_exporter:
ansible_host: 10.61.1.7 container_address: 10.90.0.9
ansible_password: VxMNsdHg6Fu5Ah9GxRTcQFVTGFRagAHD container_config:
image: prom/mysqld-exporter:latest
mysql_target_host: mysql
mysql_exporter_username: exporter
mysql_exporter_password: JmH45F3HQogkQoofLaca7YNPvYRV4KXD
fluentd:
container_address: 10.90.0.10
container_config:
image: fluentd
custom_image: yes
fluentd_syslog_port: 11514
prometheus: prometheus:
ansible_host: 10.61.1.9 container_address: 10.90.0.11
ansible_password: XPJEzq3ohbu3KxcqWnT65M9uX5Utxyx2
grafana: grafana:
ansible_host: 10.61.1.10 container_address: 10.90.0.12
ansible_password: ZsA8if5Cm4sEr299SjoMEMgqV5kBYn4a
fluentd:
ansible_host: 10.61.1.11
ansible_password: mcLjXRpVGvxk7x4NPhuHVf9n6o8nk3cS
mysql:
ansible_host: 10.61.1.12
ansible_password: cr9GvFGoqnzAdUNRvTr2DRqwQqqKLMWV

@ -0,0 +1,25 @@
- name: determine distribution
setup:
gather_subset:
- distribution
- name: install docker cli
package:
name: docker-cli
when: ansible_distribution == 'Alpine'
- block:
- name: add docker repository
include_tasks: add_docker_repo.yml
- name: install docker cli
apt:
name:
- docker-ce-cli
force_apt_get: yes
state: latest
when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'

@ -79,9 +79,7 @@
hostname: "{{ inventory_hostname }}" hostname: "{{ inventory_hostname }}"
command_handling: correct command_handling: correct
network_mode: bridge network_mode: bridge
networks: networks: "{{ [] if container_address is not defined else [{'name': 'network', 'ipv4_address': container_address}] }}"
- name: network
ipv4_address: "{{ ansible_host }}"
log_driver: local log_driver: local
detach: yes detach: yes
restart_policy: unless-stopped restart_policy: unless-stopped
@ -91,5 +89,18 @@
privileged: "{{ container_cfg.privileged | d(false) }}" privileged: "{{ container_cfg.privileged | d(false) }}"
devices: "{{ omit if container_cfg.devices is not defined else ([container_cfg.devices] | flatten(levels=1)) }}" devices: "{{ omit if container_cfg.devices is not defined else ([container_cfg.devices] | flatten(levels=1)) }}"
command: "{{ container_cfg.command | d(omit) }}" command: "{{ container_cfg.command | d(omit) }}"
env: "{{ container_cfg.env | d(omit) }}"
exposed_ports: "{{ container_cfg.exposed_ports | d(omit) }}"
delegate_to: "{{ hypervisor_hostname }}" delegate_to: "{{ hypervisor_hostname }}"
- block:
- name: ping container
ping:
rescue:
- name: install python in container
raw: apk add python3
when: container_post_build | d(false) == true

@ -0,0 +1,118 @@
- name: set container mount dirs
set_fact:
container_config_mount: /fluentd/etc/
- name: set additional exposed ports
set_fact:
container_role_config:
exposed_ports: "{{ fluentd_syslog_port }}"
when: fluentd_syslog_port is defined
- name: set fluentd sections
set_fact:
fluentd_filters:
- __postfix: 'nginx.**'
_type: parser
key_name: message
parse:
_type: json
json_parser: yajl
fluentd_matches:
- __postfix: 'nginx.**'
_type: sql
adapter: mysql2
_log_level: trace
port: 3306
host: "{{ hostvars['mysql']['container_address'] | d(hostvars['mysql']['ansible_host']) }}"
database: "{{ fluentd_database_name }}"
username: "{{ fluentd_database_user }}"
password: "{{ fluentd_database_pass }}"
flush_interval: 1s
table:
table: "{{ fluentd_database_table }}"
column_mapping: "time_iso8601:time_iso8601,\
request_time:request_time,\
upstream_response_time:upstream_response_time,\
remote_addr:remote_addr,\
remote_user:remote_user,\
time_local:time_local,\
request:request,\
status:status,\
body_bytes_sent:body_bytes_sent,\
http_referer:http_referer,\
http_user_agent:http_user_agent,\
geoip_country_code:geoip_country_code"
- block:
- name: initialize ansible mysql connection
include_tasks: init_mysql.yml
- name: create fluentd database
mysql_db:
name: "{{ fluentd_database_name }}"
encoding: utf8
login_host: "{{ hostvars['mysql']['container_address'] | d(hostvars['mysql']['ansible_host']) }}"
login_user: "{{ hostvars['mysql']['mysql_root_username'] | d('root') }}"
login_password: "{{ hostvars['mysql']['mysql_root_password'] | d(omit) }}"
- name: create fluentd database user
mysql_user:
name: "{{ fluentd_database_user }}"
password: "{{ fluentd_database_pass }}"
plugin: mysql_native_password
host: "{{ container_address | d(ansible_host) }}"
priv: "{{ fluentd_database_name }}.*:ALL"
login_host: "{{ hostvars['mysql']['container_address'] | d(hostvars['mysql']['ansible_host']) }}"
login_user: "{{ hostvars['mysql']['mysql_root_username'] | d('root') }}"
login_password: "{{ hostvars['mysql']['mysql_root_password'] | d(omit) }}"
register: result
- name: flush privileges
mysql_query:
query: FLUSH PRIVILEGES
login_host: "{{ hostvars['mysql']['container_address'] | d(hostvars['mysql']['ansible_host']) }}"
login_user: "{{ hostvars['mysql']['mysql_root_username'] | d('root') }}"
login_password: "{{ hostvars['mysql']['mysql_root_password'] | d(omit) }}"
when: result.changed
- name: initialize mysql database
mysql_query:
query: |
CREATE TABLE IF NOT EXISTS {{ fluentd_database_table | quote }} (
time_iso8601 DATETIME,
request_time FLOAT,
upstream_response_time FLOAT,
remote_addr TEXT,
remote_user TEXT,
time_local TEXT,
request TEXT,
status INTEGER,
body_bytes_sent INTEGER,
http_referer TEXT,
http_user_agent TEXT,
geoip_country_code TEXT
);
login_db: "{{ fluentd_database_name }}"
login_host: "{{ hostvars['mysql']['container_address'] | d(hostvars['mysql']['ansible_host']) }}"
login_user: "{{ hostvars['mysql']['mysql_root_username'] | d('root') }}"
login_password: "{{ hostvars['mysql']['mysql_root_password'] | d(omit) }}"
changed_when: no
delegate_to: localhost
- name: template fluentd config
template:
src: fluentd.j2
dest: "{{ (conf_dir, 'fluent.conf') | path_join }}"
lstrip_blocks: yes
force: yes

@ -0,0 +1,41 @@
{% macro fluentd_option_block(name, values, padding = 2) -%}
{% if values is mapping -%}
{% set postfix = '' if values.__postfix is not defined else (' ' + values.__postfix) -%}
<{{ name ~ postfix}}>
{% for option in values | dict2items -%}
{% if option.key != '__postfix' -%}
{{ '' if padding == 0 else (' ' * padding) -}}
{% if option.value is mapping -%}
{{ fluentd_option_block(option.key, option.value, padding + 2) -}}
{% else -%}
{% if option.key[0] == '_' -%}
{{ '@' ~ option.key[1:] }} {{ option.value }}
{% else -%}
{{ option.key }} {{ option.value }}
{% endif -%}
{% endif -%}
{% endif -%}
{% endfor -%}
{{ '' if padding == 0 else (' ' * (padding - 2)) -}}</{{ name }}>
{% endif -%}
{% endmacro -%}
{{ fluentd_option_block('system', fluentd_system | d(None)) }}
{% for source in (fluentd_sources | d([])) + (groups['all'] | map('extract', hostvars) | selectattr('fluentd_source', 'mapping') | map(attribute='fluentd_source') | list) -%}
{{ fluentd_option_block('source', source) }}
{% endfor -%}
{% for filter in (fluentd_filters | d([])) -%}
{{ fluentd_option_block('filter', filter) }}
{% endfor -%}
{% for match in (fluentd_matches | d([])) -%}
{{ fluentd_option_block('match', match) }}
{% endfor -%}

@ -1,7 +1,4 @@
timezone: Europe/Kirov timezone: Europe/Kirov
gpg_keyrings_dir: /etc/apt/trusted.gpg.d
docker_repo_url: https://download.docker.com/linux/debian
docker_network_subnet: 10.90.0.0/16 docker_network_subnet: 10.90.0.0/16
docker_network_gateway: 10.90.0.1 docker_network_gateway: 10.90.0.1

@ -13,41 +13,8 @@
state: absent state: absent
- name: ensure apt can access https repos - name: add docker repository
apt: include_tasks: add_docker_repo.yml
name:
- ca-certificates
- curl
- gnupg
- lsb-release
force_apt_get: yes
state: latest
- name: add keyrings dir
file:
state: directory
path: "{{ gpg_keyrings_dir }}"
- name: download docker gpg key
get_url:
url: "{{ docker_repo_url ~ '/gpg' }}"
dest: "{{ gpg_keyrings_dir }}/docker.asc"
mode: a+r
- name: add apt repo
apt_repository:
repo: "deb [arch=amd64 signed-by={{ (gpg_keyrings_dir ~ '/docker.asc') | quote }}] \
{{ docker_repo_url }} {{ ansible_distribution_release }} stable"
- name: update repository index
apt:
force_apt_get: yes
update_cache: yes
changed_when: no
- name: install docker and dependencies - name: install docker and dependencies

@ -0,0 +1,12 @@
- name: set container cfg
set_fact:
container_logs_mount: /var/lib/mysql/mysql-bin
container_role_config:
env:
MYSQL_ROOT_PASSWORD: "{{ mysql_root_password }}"
- name: create mysql dir
file:
path: /opt/mariadb
state: directory

@ -0,0 +1,37 @@
- name: fail if exporter host is missing
fail:
msg: mysql_target_host is not string
when: mysql_target_host is not string
- block:
- include_tasks: init_mysql.yml
- name: create exporter user
mysql_user:
name: "{{ mysql_exporter_username }}"
password: "{{ mysql_exporter_password }}"
plugin: mysql_native_password
host: "{{ container_address | d(ansible_host) }}"
resource_limits:
MAX_USER_CONNECTIONS: 3
priv:
'*.*': 'SELECT,BINLOG MONITOR,PROCESS'
'performance_schema.*': SELECT
login_host: "{{ hostvars[mysql_target_host]['container_address'] | d(hostvars[mysql_target_host]['ansible_host']) }}"
login_user: "{{ hostvars[mysql_target_host]['mysql_root_username'] | d('root') }}"
login_password: "{{ hostvars[mysql_target_host]['mysql_root_password'] | d(omit) }}"
register: result
- name: flush privileges
mysql_query:
query: FLUSH PRIVILEGES
login_host: "{{ hostvars[mysql_target_host]['container_address'] | d(hostvars[mysql_target_host]['ansible_host']) }}"
login_user: "{{ hostvars[mysql_target_host]['mysql_root_username'] | d('root') }}"
login_password: "{{ hostvars[mysql_target_host]['mysql_root_password'] | d(omit) }}"
when: result.changed
delegate_to: localhost

@ -0,0 +1,7 @@
- name: set container cfg
set_fact:
container_role_config:
env:
DATA_SOURCE_NAME: "{{ mysql_exporter_username ~ ':' ~ mysql_exporter_password ~ '@tcp(' ~
(hostvars[mysql_target_host]['container_address'] | d(hostvars[mysql_target_host]['ansible_host'])) ~
':3306)/' }}"

@ -28,7 +28,6 @@ events {
http { http {
{{ nginx_option_block(nginx_cfg.http) }} {{ nginx_option_block(nginx_cfg.http) }}
access_log /var/log/nginx/access.log custom;
include {{ (container_config_mount, 'mime.types') | path_join | quote}}; include {{ (container_config_mount, 'mime.types') | path_join | quote}};
geoip2 {{ (container_config_mount, 'geoip.mmdb') | path_join | quote }} { geoip2 {{ (container_config_mount, 'geoip.mmdb') | path_join | quote }} {
@ -36,6 +35,8 @@ http {
$geoip_country_code default=RU source=$remote_addr country iso_code; $geoip_country_code default=RU source=$remote_addr country iso_code;
} }
{{ nginx_option_block(nginx_cfg.http_post) }}
server { server {
listen 80; listen 80;
listen [::]:80; listen [::]:80;
@ -43,7 +44,7 @@ http {
server_name apache1.local; server_name apache1.local;
location / { location / {
proxy_pass http://{{ hostvars['apache1']['ansible_host'] }}; proxy_pass http://{{ hostvars['apache1']['container_address'] | d(hostvars['apache1']['ansible_host']) }};
proxy_set_header Host $proxy_host; proxy_set_header Host $proxy_host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@ -58,7 +59,7 @@ http {
server_name apache2.local; server_name apache2.local;
location / { location / {
proxy_pass http://{{ hostvars['apache2']['ansible_host'] }}; proxy_pass http://{{ hostvars['apache2']['container_address'] | d(hostvars['apache2']['ansible_host']) }};
proxy_set_header Host $proxy_host; proxy_set_header Host $proxy_host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@ -78,7 +79,7 @@ http {
location /stub_status { location /stub_status {
stub_status; stub_status;
# deny all; # deny all;
allow {{ hostvars['nginx_exporter']['ansible_host'] }}; allow {{ hostvars['nginx_exporter']['container_address'] | d(hostvars['nginx_exporter']['ansible_host']) }};
} }
{% endif -%} {% endif -%}
} }

@ -1,4 +1,4 @@
- name: set container cfg - name: set container cfg
set_fact: set_fact:
container_role_config: container_role_config:
command: "-nginx.scrape-uri=http://{{ hostvars['nginx']['ansible_host'] }}/stub_status" command: "-nginx.scrape-uri=http://{{ hostvars['nginx']['container_address'] | d(hostvars['nginx']['ansible_host']) }}/stub_status"

@ -0,0 +1,44 @@
- name: determine distribution
setup:
gather_subset:
- distribution
- block:
- name: ensure apt can access https repos
apt:
name:
- ca-certificates
- curl
- gnupg
- lsb-release
force_apt_get: yes
state: latest
- name: add keyrings dir
file:
state: directory
path: "{{ gpg_keyrings_dir }}"
- name: download docker gpg key
get_url:
url: "{{ docker_repo_url ~ '/gpg' }}"
dest: "{{ gpg_keyrings_dir }}/docker.asc"
mode: a+r
- name: add apt repo
apt_repository:
repo: "deb [arch=amd64 signed-by={{ (gpg_keyrings_dir ~ '/docker.asc') | quote }}] \
{{ docker_repo_url }} {{ ansible_distribution_release }} stable"
- name: update repository index
apt:
force_apt_get: yes
update_cache: yes
changed_when: no
when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'

@ -0,0 +1,13 @@
- name: initialize mysql connection on ansible controller
block:
- name: get ansible controller os distribution
setup:
gather_subset:
- distribution
- name: install pymysql
package:
name: "{{ { 'alpine': 'py3-pymysql', 'debian': 'python3-pymysql' }[ansible_distribution | lower] | d('pymysql') }}"
delegate_to: localhost

@ -0,0 +1,14 @@
FROM fluent/fluentd:edge
USER root
RUN apk add --no-cache --update --virtual .build-deps build-base ruby-dev && \
apk add mariadb-dev mariadb-connector-c mariadb-connector-c-dev && \
gem install mysql2 && \
gem install fluent-plugin-record-modifier && \
gem install fluent-plugin-sql && \
gem sources --clear-all && \
apk del .build-deps && \
rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
USER fluent
Loading…
Cancel
Save