以下の続きです。今度はElastic StackをデプロイしてPASのログを転送します。
- Pivotal Application Service (PAS) 2.4をCLIでAWSにインストールするメモ
- Pivotal Application Service (PAS) on AWSをPrometheusでモニタリング
前記事までの環境を前提にしています。
Elastic Stack用のLoad Balancerの設定
# target groupの追加
cat <<'EOF' >> template/modules/bosh/lbs.tf
resource "aws_lb_target_group" "elasticsearch" {
name = "${var.env_name}-elasticsearch"
port = "443"
protocol = "HTTPS"
vpc_id = "${var.vpc_id}"
health_check {
protocol = "HTTPS"
path = "/"
port = 443
matcher = "401"
healthy_threshold = 6
unhealthy_threshold = 3
timeout = 3
interval = 5
}
}
resource "aws_lb_target_group" "kibana" {
name = "${var.env_name}-kibana"
port = "443"
protocol = "HTTPS"
vpc_id = "${var.vpc_id}"
health_check {
protocol = "HTTPS"
path = "/"
port = 443
matcher = "401"
healthy_threshold = 6
unhealthy_threshold = 3
timeout = 3
interval = 5
}
}
resource "aws_lb_listener_rule" "elasticsearch" {
listener_arn = "${aws_lb_listener.bosh-lb.arn}"
priority = 27
action {
type = "forward"
target_group_arn = "${aws_lb_target_group.elasticsearch.arn}"
}
condition {
field = "host-header"
values = ["elasticsearch.sys.${var.env_name}.${var.dns_suffix}"]
}
}
resource "aws_lb_listener_rule" "kibana" {
listener_arn = "${aws_lb_listener.bosh-lb.arn}"
priority = 26
action {
type = "forward"
target_group_arn = "${aws_lb_target_group.kibana.arn}"
}
condition {
field = "host-header"
values = ["kibana.sys.${var.env_name}.${var.dns_suffix}"]
}
}
EOF
# DNSの追加
cat <<'EOF' >> template/modules/bosh/dns.tf
resource "aws_route53_record" "elasticsearch" {
zone_id = "${var.zone_id}"
name = "elasticsearch.sys.${var.env_name}.${var.dns_suffix}"
type = "A"
alias {
name = "${aws_lb.bosh-lb.dns_name}"
zone_id = "${aws_lb.bosh-lb.zone_id}"
evaluate_target_health = true
}
}
resource "aws_route53_record" "kibana" {
zone_id = "${var.zone_id}"
name = "kibana.sys.${var.env_name}.${var.dns_suffix}"
type = "A"
alias {
name = "${aws_lb.bosh-lb.dns_name}"
zone_id = "${aws_lb.bosh-lb.zone_id}"
evaluate_target_health = true
}
}
EOF
# outputsの追加
cat <<'EOF' >> template/modules/bosh/outputs.tf
output "elasticsearch_target_groups" {
value = [
"${aws_lb_target_group.elasticsearch.name}"
]
}
output "kibana_target_groups" {
value = [
"${aws_lb_target_group.kibana.name}"
]
}
EOF
cat <<'EOF' >> template/terraforming-pas/outputs.tf
output "elasticsearch_target_groups" {
value = "${module.bosh.elasticsearch_target_groups}"
}
output "kibana_target_groups" {
value = "${module.bosh.kibana_target_groups}"
}
EOF
terraform init template/terraforming-pas
terraform plan -out plan template/terraforming-pas
terraform apply plan
export ELASTICSEARCH_TARGET_GROUPS="[$(terraform output elasticsearch_target_groups | tr -d '\n')]"
export KIBANA_TARGET_GROUPS="[$(terraform output kibana_target_groups | tr -d '\n')]"
cat <<EOF >> cloud-config.yml
- name: elasticsearch-lb
cloud_properties:
lb_target_groups: ${ELASTICSEARCH_TARGET_GROUPS}
- name: kibana-lb
cloud_properties:
lb_target_groups: ${KIBANA_TARGET_GROUPS}
EOF
scp -i opsman.pem -o "StrictHostKeyChecking=no" cloud-config.yml ubuntu@${OM_TARGET}:~/bosh-manifests/
./ssh-opsman.sh
cd ~/bosh-manifests
bosh update-config --type=cloud --name=bosh cloud-config.yml
Elastic Stackのデプロイ
1709c958bced84d9971a5cc54c0096506a6a6c74
cd ~/bosh-manifests
git submodule add https://github.com/bosh-elastic-stack/elastic-stack-bosh-deployment.git
wget https://github.com/Pivotal-Japan/demo-bosh-manifests/raw/master/logstash.conf
cat <<'EOF' > deploy-elastic-stack.sh
#!/bin/bash
bosh -d elastic-stack deploy ./elastic-stack-bosh-deployment/elastic-stack.yml \
-l ./elastic-stack-bosh-deployment/versions.yml \
-o ./elastic-stack-bosh-deployment/ops-files/vm_types.yml \
-o ./elastic-stack-bosh-deployment/ops-files/disk_types.yml \
-o ./elastic-stack-bosh-deployment/ops-files/instances.yml \
-o ./elastic-stack-bosh-deployment/ops-files/networks.yml \
-o ./elastic-stack-bosh-deployment/ops-files/azs.yml \
-o ./elastic-stack-bosh-deployment/ops-files/elasticsearch-https-and-basic-auth.yml \
-o ./elastic-stack-bosh-deployment/ops-files/elasticsearch-add-lb.yml \
-o ./elastic-stack-bosh-deployment/ops-files/elasticsearch-allow-ingest.yml \
-o ./elastic-stack-bosh-deployment/ops-files/logstash-readiness-probe.yml \
-o ./elastic-stack-bosh-deployment/ops-files/logstash-tls.yml \
-o ./elastic-stack-bosh-deployment/ops-files/logstash-elasticsearch-https.yml \
-o ./elastic-stack-bosh-deployment/ops-files/logstash-elasticsearch-basic-auth.yml \
-o ./elastic-stack-bosh-deployment/ops-files/logstash-persistent-queue.yml \
-o ./elastic-stack-bosh-deployment/ops-files/kibana-https-and-basic-auth.yml \
-o ./elastic-stack-bosh-deployment/ops-files/kibana-elasticsearch-https.yml \
-o ./elastic-stack-bosh-deployment/ops-files/kibana-elasticsearch-basic-auth.yml \
-o ./elastic-stack-bosh-deployment/ops-files/kibana-add-lb.yml \
-o ./elastic-stack-bosh-deployment/ops-files/elasticsearch-share-link.yml \
--var-file logstash.conf=logstash.conf \
-v elasticsearch_master_instances=1 \
-v elasticsearch_master_vm_type=m4.large \
-v elasticsearch_master_disk_type=10240 \
-v elasticsearch_master_network=bosh \
-v elasticsearch_master_azs="[ap-northeast-1a, ap-northeast-1c, ap-northeast-1d]" \
-v elasticsearch_username=admin \
-v logstash_instances=1 \
-v logstash_vm_type=t2.medium \
-v logstash_disk_type=5120 \
-v logstash_network=bosh \
-v logstash_azs="[ap-northeast-1a, ap-northeast-1c, ap-northeast-1d]" \
-v logstash_readiness_probe_http_port=0 \
-v logstash_readiness_probe_tcp_port=5514 \
-v logstash_queue_max_bytes=1g \
-v kibana_instances=1 \
-v kibana_vm_type=t2.micro \
-v kibana_network=bosh \
-v kibana_azs="[ap-northeast-1a, ap-northeast-1c, ap-northeast-1d]" \
-v kibana_username=admin \
-v kibana_elasticsearch_ssl_verification_mode=none \
-v logstash_ip=10.0.20.200 \
-o <(cat <<EOF
# custom ops-files
- type: replace
path: /instance_groups/name=logstash/networks/0/static_ips?
value:
- ((logstash_ip))
- type: replace
path: /variables/name=logstash_tls/options/alternative_names
value:
- ((logstash_ip))
- logstash.service.bosh.internal
# vm_extentions (spot instance)
- type: replace
path: /instance_groups/name=elasticsearch-master/vm_extensions?/-
value: spot-instance-m4-large
- type: replace
path: /instance_groups/name=kibana/vm_extensions?/-
value: spot-instance-t2-micro
- type: replace
path: /instance_groups/name=logstash/vm_extensions?/-
value: spot-instance-t2-medium
EOF) \
--no-redact \
$@ \
EOF
chmod +x deploy-elastic-stack.sh
./deploy-elastic-stack.sh
Continue? [yN]: y
Task 862
Task 862 | 16:43:43 | Preparing deployment: Preparing deployment (00:00:04)
Task 862 | 16:43:49 | Preparing package compilation: Finding packages to compile (00:00:00)
Task 862 | 16:43:49 | Compiling packages: kibana/94bf796d73d8fd65a1aa5a175e236b5637729d419b5fe7a7311c40d6bc990192
Task 862 | 16:43:49 | Compiling packages: python2.7/516450abf69ffb0981b597875f1fbcf357b92a19a4f0690c9b73e56d94224aef
Task 862 | 16:43:49 | Compiling packages: logstash/604098c6f84527d97608c0fddfabca72941f29447fa6e5f2824a84bbe3f50d63
Task 862 | 16:43:49 | Compiling packages: nginx/23c0391f6bb6630cf68ba02c99f93eabdd65839d
Task 862 | 16:45:20 | Compiling packages: logstash/604098c6f84527d97608c0fddfabca72941f29447fa6e5f2824a84bbe3f50d63 (00:01:31)
Task 862 | 16:45:20 | Compiling packages: elasticsearch/13dba42e3b47fdfb4dda5b1234ef280fc7f464b6ad0b5d152ae2f685dab547ca (00:00:10)
Task 862 | 16:45:30 | Compiling packages: java/0ab4370b61ce3a2b28a73718dfd608dc0f393678
Task 862 | 16:45:38 | Compiling packages: kibana/94bf796d73d8fd65a1aa5a175e236b5637729d419b5fe7a7311c40d6bc990192 (00:01:49)
Task 862 | 16:45:44 | Compiling packages: java/0ab4370b61ce3a2b28a73718dfd608dc0f393678 (00:00:14)
Task 862 | 16:46:06 | Compiling packages: nginx/23c0391f6bb6630cf68ba02c99f93eabdd65839d (00:02:17)
Task 862 | 16:46:18 | Compiling packages: python2.7/516450abf69ffb0981b597875f1fbcf357b92a19a4f0690c9b73e56d94224aef (00:02:29)
Task 862 | 16:46:58 | Creating missing vms: elasticsearch-master/46ab4227-12d2-47ff-8b0d-8fc0f8ab872b (0)
Task 862 | 16:46:58 | Creating missing vms: logstash/e78dd167-90d7-43a6-8cb0-395f707f4b33 (0)
Task 862 | 16:46:58 | Creating missing vms: kibana/dd8a1a32-52da-4967-aa03-c0a4fbc83c9e (0) (00:01:06)
Task 862 | 16:48:06 | Creating missing vms: elasticsearch-master/46ab4227-12d2-47ff-8b0d-8fc0f8ab872b (0) (00:01:08)
Task 862 | 16:48:13 | Creating missing vms: logstash/e78dd167-90d7-43a6-8cb0-395f707f4b33 (0) (00:01:15)
Task 862 | 16:48:13 | Updating instance elasticsearch-master: elasticsearch-master/46ab4227-12d2-47ff-8b0d-8fc0f8ab872b (0) (canary) (00:01:14)
Task 862 | 16:49:27 | Updating instance logstash: logstash/e78dd167-90d7-43a6-8cb0-395f707f4b33 (0) (canary) (00:01:35)
Task 862 | 16:51:02 | Updating instance kibana: kibana/dd8a1a32-52da-4967-aa03-c0a4fbc83c9e (0) (canary) (00:00:45)
Task 862 Started Mon Mar 25 16:43:43 UTC 2019
Task 862 Finished Mon Mar 25 16:51:47 UTC 2019
Task 862 Duration 00:08:04
Task 862 done
Succeeded
bosh vms
Using environment '10.0.16.5' as client 'ops_manager'
Task 878
Task 879
Task 880
Task 878 done
Task 880 done
Task 879 done
Deployment 'cf-013bf999f314121d05fc'
Instance Process State AZ IPs VM CID VM Type Active
clock_global/29b2abe0-b6aa-4f6d-975f-22fd828fa699 running ap-northeast-1a 10.0.4.12 i-09bd71be81d273774 t2.medium true
cloud_controller/0780e116-554b-4159-a30c-bf19f26a4481 running ap-northeast-1a 10.0.4.10 i-0dc73ef4cd7e70cfb t2.medium true
cloud_controller_worker/15603b1a-1b0a-4edf-9240-8d06087121be running ap-northeast-1a 10.0.4.13 i-028566ae524c2cd5c t2.micro true
credhub/f516b3f6-44e3-476e-bc9e-d062c7be2279 running ap-northeast-1a 10.0.4.20 i-08d7a63eaba917254 m4.large true
diego_brain/9f15438e-3312-4754-85e1-c5e58b398fe2 running ap-northeast-1a 10.0.4.14 i-03e665bfeaf2f30a4 t2.micro true
diego_cell/65728f05-d3ea-4782-aba7-b6cd070820c4 running ap-northeast-1a 10.0.4.15 i-087f2b2e9347f2487 r4.xlarge true
diego_database/01457cd8-f05a-4eed-88db-bd64058f473e running ap-northeast-1a 10.0.4.8 i-081dde85a8e20c9f3 t2.micro true
doppler/c4b413c7-d6f8-4d0d-81af-ac0ff8495785 running ap-northeast-1a 10.0.4.19 i-013fd406f9ad8807f t2.medium true
loggregator_trafficcontroller/36f52a7e-1240-4506-b6da-038442c9ff97 running ap-northeast-1a 10.0.4.16 i-063ef7d7381e64290 t2.micro true
mysql/72bb8162-44cd-4c96-92c0-b44410a4e3b6 running ap-northeast-1a 10.0.4.7 i-000689bb28df9a112 m4.large true
mysql_proxy/77261bd3-dda7-4813-9928-fa30054c01a4 running ap-northeast-1a 10.0.4.6 i-066ff7b03150440e1 t2.micro true
nats/6b4cb4e8-f81c-4f03-aa3f-9bba4d82496a running ap-northeast-1a 10.0.4.5 i-087ff327ee56eeb57 t2.micro true
router/4e54efdd-0387-473b-bc97-1200be6a6659 running ap-northeast-1a 10.0.4.11 i-03199618ff2d5bd12 t2.micro true
syslog_adapter/870032c8-7c4a-49fa-923a-61519e5a93fa running ap-northeast-1a 10.0.4.17 i-0a035fc92e7458ea5 t2.micro true
syslog_scheduler/416f7792-7c89-4b09-b2c8-677cabcfd3a1 running ap-northeast-1a 10.0.4.18 i-0ce495fb2370f69c2 t2.micro true
uaa/c898e032-0e23-444f-9e93-ed3a8c2c9542 running ap-northeast-1a 10.0.4.9 i-0cca8daa56f06e04e t2.medium true
16 vms
Deployment 'elastic-stack'
Instance Process State AZ IPs VM CID VM Type Active
elasticsearch-master/46ab4227-12d2-47ff-8b0d-8fc0f8ab872b running ap-northeast-1a 10.0.20.5 i-0a55336ff0066af2b m4.large true
kibana/dd8a1a32-52da-4967-aa03-c0a4fbc83c9e running ap-northeast-1a 10.0.20.6 i-09b36fc27b068c871 t2.micro true
logstash/e78dd167-90d7-43a6-8cb0-395f707f4b33 running ap-northeast-1a 10.0.20.200 i-0a640dbae14092686 t2.medium true
3 vms
Deployment 'prometheus'
Instance Process State AZ IPs VM CID VM Type Active
alertmanager/ef31bab1-e6ba-4196-86de-1c265a7d18ed running ap-northeast-1c 10.0.21.5 i-07a997f6b646aab17 t2.micro true
firehose/dafb1c09-97ef-46de-9fac-95b2b7ad0601 running ap-northeast-1c 10.0.21.9 i-0d76fbd0ccc718bfb t2.micro true
grafana/9e8867f7-686d-414c-8814-b0963f41fd91 running ap-northeast-1c 10.0.21.7 i-09d6c2a41bb8c365e t2.micro true
nginx/e834eeb3-6fba-413b-a5c0-24cf9d070f27 running ap-northeast-1c 10.0.21.8 i-07c40cd12af29d81a t2.micro true
prometheus2/6ddd5e0f-e97d-4b9b-b6b8-a138ccb55d4b running ap-northeast-1c 10.0.21.6 i-05e9907916cd8015f t2.small true
5 vms
Succeeded
PlantUML(参考)
@startuml
package "public" {
package "az1 (10.0.0.0/24)" {
node "Ops Manager"
rectangle "web-lb-1"
rectangle "ssh-lb-1"
rectangle "bosh-lb-1"
boundary "NAT Gateway"
}
package "az2 (10.0.1.0/24)" {
rectangle "web-lb-2"
rectangle "ssh-lb-2"
rectangle "bosh-lb-2"
}
package "az3 (10.0.2.0/24)" {
rectangle "web-lb-3"
rectangle "ssh-lb-3"
rectangle "bosh-lb-3"
}
}
package "infrastructure" {
package "az1 (10.0.16.0/28)" {
node "BOSH Director"
}
}
package "deployment" {
package "az1 (10.0.4.0/24)" {
node "NATS"
node "Router"
database "File Storage"
package "MySQL" {
node "MySQL Proxy"
database "MySQL Server"
}
package "CAPI" {
node "Cloud Controller"
node "Clock Global"
node "Cloud Controller Worker"
}
package "Diego" {
node "Diego Brain"
node "DiegoCell" {
(app3)
(app2)
(app1)
}
node "Diego BBS"
}
package "Loggregator" {
node "Loggregator Trafficcontroller"
node "Syslog Adapter"
node "Syslog Scheduler"
node "Doppler Server"
}
node "UAA"
node "CredHub"
}
}
package "bosh" {
package "az1 (10.0.20.0/24)" {
node "Elasticsearch" {
(elasticsearch)
(nginx_e)
}
node "Kibana" {
(kibana)
(nginx_k)
}
node "Logstash"
}
package "az2 (10.0.21.0/24)" {
node "Nginx"
node "Prometheus2" {
(prometheus2)
(bosh exporter)
(cf exporter)
}
node "AlertManager"
node "Grafana"
node "Firehose Exporter"
}
package "az3 (10.0.22.0/24)" {
}
}
boundary "Internet Gateway"
actor User #red
actor Developer #blue
actor Operator #green
User -[#red]--> [web-lb-1]
User -[#red]--> [web-lb-2]
User -[#red]--> [web-lb-3]
Developer -[#blue]--> [web-lb-1] : "cf push"
Developer -[#blue]--> [web-lb-2]
Developer -[#blue]--> [web-lb-3]
Developer -[#magenta]--> [ssh-lb-1] : "cf ssh"
Developer -[#magenta]--> [ssh-lb-2]
Developer -[#magenta]--> [ssh-lb-3]
Operator -[#green]--> [Ops Manager]
Operator -[#green]--> [bosh-lb-1]
Operator -[#green]--> [bosh-lb-2]
Operator -[#green]--> [bosh-lb-3]
public -up-> [Internet Gateway]
infrastructure -> [NAT Gateway]
deployment -> [NAT Gateway]
[Ops Manager] .> [BOSH Director] :bosh
[web-lb-1] -[#red]-> Router
[web-lb-1] -[#blue]-> Router
[web-lb-2] -[#red]-> Router
[web-lb-2] -[#blue]-> Router
[web-lb-3] -[#red]-> Router
[web-lb-3] -[#blue]-> Router
[ssh-lb-1] -[#magenta]-> [Diego Brain]
[ssh-lb-2] -[#magenta]-> [Diego Brain]
[ssh-lb-3] -[#magenta]-> [Diego Brain]
[bosh-lb-1] -[#green]-> [Nginx]
[bosh-lb-2] -[#green]-> [Nginx]
[bosh-lb-3] -[#green]-> [Nginx]
[bosh-lb-1] -[#green]-> [nginx_e]
[bosh-lb-2] -[#green]-> [nginx_e]
[bosh-lb-3] -[#green]-> [nginx_e]
[bosh-lb-1] -[#green]-> [nginx_k]
[bosh-lb-2] -[#green]-> [nginx_k]
[bosh-lb-3] -[#green]-> [nginx_k]
Router -[#red]-> app1
Router -[#blue]-> [Cloud Controller]
Router -[#blue]-> [UAA]
[Doppler Server] --> [Loggregator Trafficcontroller]
[Loggregator Trafficcontroller] -right-> [Syslog Adapter]
[Syslog Adapter] -up-> [Syslog Scheduler]
[Cloud Controller] --> [MySQL Proxy]
[Firehose Exporter] -up-> [Loggregator Trafficcontroller]
[cf exporter] -up-> [Cloud Controller]
[bosh exporter] -up-> [BOSH Director]
[prometheus2] .> [Firehose Exporter] : scrape
[prometheus2] .> [cf exporter] : scrape
[prometheus2] .> [bosh exporter] : scrape
[Grafana] -down-> [prometheus2]
[prometheus2] -down-> [AlertManager]
[Nginx] -[#green]-> [prometheus2]
[Nginx] -[#green]-> [AlertManager]
[Nginx] -[#green]-> [Grafana]
[kibana] --> [elasticsearch]
[Logstash] -> [elasticsearch]
[nginx_e] -> [elasticsearch]
[nginx_k] -> [kibana]
Diego .> [Doppler Server] : metrics
CAPI .> [Doppler Server] : metrics
Router .> [Doppler Server] : metrics
app1 ..> [Doppler Server] : log&metrics
app2 ..> [Doppler Server] : log&metrics
app3 ..> [Doppler Server] : log&metrics
@enduml
./credhub-login.sh
credhub get -n /p-bosh/elastic-stack/kibana_password
./credhub-login.sh
credhub get -n /p-bosh/elastic-stack/elasticsearch_password
Firehose to SyslogでPASのアプリログをLogstashに転送
cat <<'EOF' > uaac-create-client-firehose-to-syslog.sh
#!/bin/bash
# use ${BOSH_CLIENT_SECRET} for convenience
uaac client add firehose-to-syslog \
--scope uaa.none \
--authorized_grant_types client_credentials,refresh_token \
--authorities doppler.firehose,cloud_controller.global_auditor \
-s ${BOSH_CLIENT_SECRET}
EOF
chmod +x uaac-create-client-firehose-to-syslog.sh
./uaac-token-client-get-pas.sh
./uaac-create-client-firehose-to-syslog.sh
scope: uaa.none
client_id: firehose-to-syslog
resource_ids: none
authorized_grant_types: refresh_token client_credentials
autoapprove:
authorities: cloud_controller.global_auditor doppler.firehose
name: firehose-to-syslog
required_user_groups:
lastmodified: 1553533575000
id: firehose-to-syslog
./credhub-login.sh
credhub get -n /p-bosh/elastic-stack/logstash_tls | bosh int - --path /value/ca > logstash_ca.pem
ADMIN_PASSWORD=$(om credentials -p cf -c .uaa.admin_credentials --format json | jq -r .password)
API_URL=https://api.$(terraform output sys_domain)
cf login -a ${API_URL} -u admin -p ${ADMIN_PASSWORD}
cf target -o system
cf create-space firehose-to-syslog
cf target -s firehose-to-syslog
mkdir firehose-to-syslog
wget https://github.com/cloudfoundry-community/firehose-to-syslog/releases/download/5.1.0/firehose-to-syslog_linux_amd64 -P firehose-to-syslog
chmod +x ./firehose-to-syslog/firehose-to-syslog_linux_amd64
scp -i opsman.pem -o "StrictHostKeyChecking=no" ubuntu@${OM_TARGET}:~/bosh-manifests/logstash_ca.pem firehose-to-syslog/
cd firehose-to-syslog
cat <<'EOF' > manifest.yml
applications:
- name: firehose-to-syslog
memory: 256m
buildpack: binary_buildpack
command: ./firehose-to-syslog_linux_amd64
routes:
- route: firehose-to-syslog.((system_domain))
env:
API_ENDPOINT: https://api.((system_domain))
DEBUG: false
DOPPLER_ENDPOINT: wss://doppler.((system_domain)):((doppler_port))
EVENTS: LogMessage,Error,HttpStartStop
FIREHOSE_CLIENT_ID: firehose-to-syslog
FIREHOSE_CLIENT_SECRET: ((client_secret))
FIREHOSE_SUBSCRIPTION_ID: firehose-to-syslog
LOG_EVENT_TOTALS: true
LOG_EVENT_TOTALS_TIME: 10s
SKIP_SSL_VALIDATION: true
SYSLOG_ENDPOINT: ((logstash_ip)):5514
SYSLOG_PROTOCOL: tcp+tls
CERT_PEM: logstash_ca.pem
ENABLE_STATS_SERVER: true
CF_PULL_TIME: 120s
EOF
export FIREHOSE_TO_SYSLOG_CLIENT_SECRET=$(om curl -s -p "/api/v0/deployed/director/credentials/bosh_commandline_credentials" | jq -r '.credential' | sed 's/ /\
/g' | grep BOSH_CLIENT_SECRET | sed 's/BOSH_CLIENT_SECRET=//g')
cf push \
--var system_domain=$(terraform output --state=../terraform.tfstate sys_domain) \
--var logstash_ip=10.0.20.200 \
--var client_secret=${FIREHOSE_TO_SYSLOG_CLIENT_SECRET} \
--var doppler_port=443
cd ..
PlantUML(参考)
@startuml
package "public" {
package "az1 (10.0.0.0/24)" {
node "Ops Manager"
rectangle "web-lb-1"
rectangle "ssh-lb-1"
rectangle "bosh-lb-1"
boundary "NAT Gateway"
}
package "az2 (10.0.1.0/24)" {
rectangle "web-lb-2"
rectangle "ssh-lb-2"
rectangle "bosh-lb-2"
}
package "az3 (10.0.2.0/24)" {
rectangle "web-lb-3"
rectangle "ssh-lb-3"
rectangle "bosh-lb-3"
}
}
package "infrastructure" {
package "az1 (10.0.16.0/28)" {
node "BOSH Director"
}
}
package "deployment" {
package "az1 (10.0.4.0/24)" {
node "NATS"
node "Router"
database "File Storage"
package "MySQL" {
node "MySQL Proxy"
database "MySQL Server"
}
package "CAPI" {
node "Cloud Controller"
node "Clock Global"
node "Cloud Controller Worker"
}
package "Diego" {
node "Diego Brain"
node "DiegoCell" {
(app3)
(app2)
(app1)
(firehose-to-syslog)
}
node "Diego BBS"
}
package "Loggregator" {
node "Loggregator Trafficcontroller"
node "Syslog Adapter"
node "Syslog Scheduler"
node "Doppler Server"
}
node "UAA"
node "CredHub"
}
}
package "bosh" {
package "az1 (10.0.20.0/24)" {
node "Elasticsearch" {
(elasticsearch)
(nginx_e)
}
node "Kibana" {
(kibana)
(nginx_k)
}
node "Logstash"
}
package "az2 (10.0.21.0/24)" {
node "Nginx"
node "Prometheus2" {
(prometheus2)
(bosh exporter)
(cf exporter)
}
node "AlertManager"
node "Grafana"
node "Firehose Exporter"
}
package "az3 (10.0.22.0/24)" {
}
}
boundary "Internet Gateway"
actor User #red
actor Developer #blue
actor Operator #green
User -[#red]--> [web-lb-1]
User -[#red]--> [web-lb-2]
User -[#red]--> [web-lb-3]
Developer -[#blue]--> [web-lb-1] : "cf push"
Developer -[#blue]--> [web-lb-2]
Developer -[#blue]--> [web-lb-3]
Developer -[#magenta]--> [ssh-lb-1] : "cf ssh"
Developer -[#magenta]--> [ssh-lb-2]
Developer -[#magenta]--> [ssh-lb-3]
Operator -[#green]--> [Ops Manager]
Operator -[#green]--> [bosh-lb-1]
Operator -[#green]--> [bosh-lb-2]
Operator -[#green]--> [bosh-lb-3]
public -up-> [Internet Gateway]
infrastructure -> [NAT Gateway]
deployment -> [NAT Gateway]
[Ops Manager] .> [BOSH Director] :bosh
[web-lb-1] -[#red]-> Router
[web-lb-1] -[#blue]-> Router
[web-lb-2] -[#red]-> Router
[web-lb-2] -[#blue]-> Router
[web-lb-3] -[#red]-> Router
[web-lb-3] -[#blue]-> Router
[ssh-lb-1] -[#magenta]-> [Diego Brain]
[ssh-lb-2] -[#magenta]-> [Diego Brain]
[ssh-lb-3] -[#magenta]-> [Diego Brain]
[bosh-lb-1] -[#green]-> [Nginx]
[bosh-lb-2] -[#green]-> [Nginx]
[bosh-lb-3] -[#green]-> [Nginx]
[bosh-lb-1] -[#green]-> [nginx_e]
[bosh-lb-2] -[#green]-> [nginx_e]
[bosh-lb-3] -[#green]-> [nginx_e]
[bosh-lb-1] -[#green]-> [nginx_k]
[bosh-lb-2] -[#green]-> [nginx_k]
[bosh-lb-3] -[#green]-> [nginx_k]
Router -[#red]-> app1
Router -[#blue]-> [Cloud Controller]
Router -[#blue]-> [UAA]
[Doppler Server] --> [Loggregator Trafficcontroller]
[Loggregator Trafficcontroller] -right-> [Syslog Adapter]
[Syslog Adapter] -up-> [Syslog Scheduler]
[Cloud Controller] --> [MySQL Proxy]
[Firehose Exporter] -up-> [Loggregator Trafficcontroller]
[cf exporter] -up-> [Cloud Controller]
[bosh exporter] -up-> [BOSH Director]
[firehose-to-syslog] -> [Loggregator Trafficcontroller]
[firehose-to-syslog] .> [Logstash] :syslog
[prometheus2] .> [Firehose Exporter] : scrape
[prometheus2] .> [cf exporter] : scrape
[prometheus2] .> [bosh exporter] : scrape
[Grafana] -down-> [prometheus2]
[prometheus2] -down-> [AlertManager]
[Nginx] -[#green]-> [prometheus2]
[Nginx] -[#green]-> [AlertManager]
[Nginx] -[#green]-> [Grafana]
[kibana] --> [elasticsearch]
[Logstash] -> [elasticsearch]
[nginx_e] -> [elasticsearch]
[nginx_k] -> [kibana]
Diego .> [Doppler Server] : metrics
CAPI .> [Doppler Server] : metrics
Router .> [Doppler Server] : metrics
app1 ..> [Doppler Server] : log&metrics
app2 ..> [Doppler Server] : log&metrics
app3 ..> [Doppler Server] : log&metrics
@enduml
PASのコンポーネントログをLogstashに転送
pas/config.yml
.properties.syslog_drop_debug:
value: true
.properties.syslog_host:
value: ((syslog_host))
.properties.syslog_port:
value: ((syslog_port))
.properties.syslog_protocol:
value: tcp
.properties.syslog_tls:
value: enabled
.properties.syslog_tls.enabled.tls_ca_cert:
value: ((syslog_tls_ca_cert))
.properties.syslog_tls.enabled.tls_permitted_peer:
value: ((syslog_tls_permitted_peer))
pas/vars.yml
syslog_host: 10-0-20-200.sslip.io
syslog_port: 5514
syslog_tls_permitted_peer: "*.sslip.io"
syslog_tls_ca_cert: |
-----BEGIN CERTIFICATE-----
(logstash_ca.pemの内容)
-----END CERTIFICATE-----
OpsManagerのGUIで"REVIEW PENDING CHANGES" => "APPLY CHANGES"をクリックするか、om apply-changes
コマンドを実行してPASを再デプロイしてください。
PlantUML(参考)
@startuml
package "public" {
package "az1 (10.0.0.0/24)" {
node "Ops Manager"
rectangle "web-lb-1"
rectangle "ssh-lb-1"
rectangle "bosh-lb-1"
boundary "NAT Gateway"
}
package "az2 (10.0.1.0/24)" {
rectangle "web-lb-2"
rectangle "ssh-lb-2"
rectangle "bosh-lb-2"
}
package "az3 (10.0.2.0/24)" {
rectangle "web-lb-3"
rectangle "ssh-lb-3"
rectangle "bosh-lb-3"
}
}
package "infrastructure" {
package "az1 (10.0.16.0/28)" {
node "BOSH Director"
}
}
package "deployment" {
package "az1 (10.0.4.0/24)" {
node "NATS"
node "Router"
database "File Storage"
package "MySQL" {
node "MySQL Proxy"
database "MySQL Server"
}
package "CAPI" {
node "Cloud Controller"
node "Clock Global"
node "Cloud Controller Worker"
}
package "Diego" {
node "Diego Brain"
node "DiegoCell" {
(app3)
(app2)
(app1)
(firehose-to-syslog)
}
node "Diego BBS"
}
package "Loggregator" {
node "Loggregator Trafficcontroller"
node "Syslog Adapter"
node "Syslog Scheduler"
node "Doppler Server"
}
node "UAA"
node "CredHub"
}
}
package "bosh" {
package "az1 (10.0.20.0/24)" {
node "Elasticsearch" {
(elasticsearch)
(nginx_e)
}
node "Kibana" {
(kibana)
(nginx_k)
}
node "Logstash"
}
package "az2 (10.0.21.0/24)" {
node "Nginx"
node "Prometheus2" {
(prometheus2)
(bosh exporter)
(cf exporter)
}
node "AlertManager"
node "Grafana"
node "Firehose Exporter"
}
package "az3 (10.0.22.0/24)" {
}
}
boundary "Internet Gateway"
actor User #red
actor Developer #blue
actor Operator #green
User -[#red]--> [web-lb-1]
User -[#red]--> [web-lb-2]
User -[#red]--> [web-lb-3]
Developer -[#blue]--> [web-lb-1] : "cf push"
Developer -[#blue]--> [web-lb-2]
Developer -[#blue]--> [web-lb-3]
Developer -[#magenta]--> [ssh-lb-1] : "cf ssh"
Developer -[#magenta]--> [ssh-lb-2]
Developer -[#magenta]--> [ssh-lb-3]
Operator -[#green]--> [Ops Manager]
Operator -[#green]--> [bosh-lb-1]
Operator -[#green]--> [bosh-lb-2]
Operator -[#green]--> [bosh-lb-3]
public -up-> [Internet Gateway]
infrastructure -> [NAT Gateway]
deployment -> [NAT Gateway]
[Ops Manager] .> [BOSH Director] :bosh
[web-lb-1] -[#red]-> Router
[web-lb-1] -[#blue]-> Router
[web-lb-2] -[#red]-> Router
[web-lb-2] -[#blue]-> Router
[web-lb-3] -[#red]-> Router
[web-lb-3] -[#blue]-> Router
[ssh-lb-1] -[#magenta]-> [Diego Brain]
[ssh-lb-2] -[#magenta]-> [Diego Brain]
[ssh-lb-3] -[#magenta]-> [Diego Brain]
[bosh-lb-1] -[#green]-> [Nginx]
[bosh-lb-2] -[#green]-> [Nginx]
[bosh-lb-3] -[#green]-> [Nginx]
[bosh-lb-1] -[#green]-> [nginx_e]
[bosh-lb-2] -[#green]-> [nginx_e]
[bosh-lb-3] -[#green]-> [nginx_e]
[bosh-lb-1] -[#green]-> [nginx_k]
[bosh-lb-2] -[#green]-> [nginx_k]
[bosh-lb-3] -[#green]-> [nginx_k]
Router -[#red]-> app1
Router -[#blue]-> [Cloud Controller]
Router -[#blue]-> [UAA]
[Doppler Server] --> [Loggregator Trafficcontroller]
[Loggregator Trafficcontroller] -right-> [Syslog Adapter]
[Syslog Adapter] -up-> [Syslog Scheduler]
[Cloud Controller] --> [MySQL Proxy]
[Firehose Exporter] -up-> [Loggregator Trafficcontroller]
[cf exporter] -up-> [Cloud Controller]
[bosh exporter] -up-> [BOSH Director]
[firehose-to-syslog] -> [Loggregator Trafficcontroller]
[firehose-to-syslog] .> [Logstash] :syslog
[prometheus2] .> [Firehose Exporter] : scrape
[prometheus2] .> [cf exporter] : scrape
[prometheus2] .> [bosh exporter] : scrape
[Grafana] -down-> [prometheus2]
[prometheus2] -down-> [AlertManager]
[Nginx] -[#green]-> [prometheus2]
[Nginx] -[#green]-> [AlertManager]
[Nginx] -[#green]-> [Grafana]
[kibana] --> [elasticsearch]
[Logstash] -> [elasticsearch]
[nginx_e] -> [elasticsearch]
[nginx_k] -> [kibana]
Diego .> [Doppler Server] : metrics
CAPI .> [Doppler Server] : metrics
Router .> [Doppler Server] : metrics
Diego .> [Logstash] : syslog
CAPI .> [Logstash] : syslog
Router .> [Logstash] : syslog
app1 ..> [Doppler Server] : log&metrics
app2 ..> [Doppler Server] : log&metrics
app3 ..> [Doppler Server] : log&metrics
@enduml
LogstashのIPをBOSH DNSのaliasに登録
TBD