Skip to content

Commit 60f2bb2

Browse files
bearddan2000bearddan2000
committed
initial commit
0 parents  commit 60f2bb2

17 files changed

+612
-0
lines changed

‎.env

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
STACK_VERSION=8.6.1
2+
COCKROACH_VERSION=v19.2.4

‎README.md

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# python-cli-cockroachdb-multi-node-without-ssl-to-multi-node-elasticsearch-client-pop
2+
3+
## Description
4+
Reads a multi node cluster for data in `pop-demo` document.
5+
6+
Uses `pop` table then covverts it to json for
7+
elasticsearch to use.
8+
9+
## Tech stack
10+
- python
11+
- sqlalchemy
12+
- elasticsearch
13+
- kibana
14+
- cockroach
15+
16+
## Docker stack
17+
- python
18+
- elasticsearch
19+
- kibana
20+
- cockroachdb/cockroach:v19.2.4
21+
22+
## To run
23+
`sudo ./install.sh -u`
24+
25+
## To stop (optional)
26+
`sudo ./install.sh -d`
27+
28+
## For help
29+
`sudo ./install.sh -h`
30+
31+
## Credit
32+
- [Docker setup](https://lynn-kwong.medium.com/all-you-need-to-know-about-using-elasticsearch-in-python-b9ed00e0fdf0)
33+
- [Search setup](https://www.elastic.co/guide/en/elasticsearch/client/python-api/master/examples.html)

‎db/init-data.sh

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env sh
2+
3+
4+
HOST="db"
5+
DATA_SRC=/docker-entrypoint-initdb.d
6+
hold=/tmp/hold.sql
7+
8+
/cockroach/cockroach.sh init --host "$HOST" --insecure
9+
10+
# /cockroach/cockroach.sh user set testuser --host "$HOST" --insecure
11+
12+
for sql in `ls ${DATA_SRC}/*.sql`; do
13+
cat $sql >> $hold
14+
done
15+
16+
cat $hold | /cockroach/cockroach.sh sql --host $HOST --insecure
17+
18+
exit 0

‎db/sql/00-create-table.sql

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
CREATE DATABASE IF NOT EXISTS beverage;
2+
3+
USE beverage;
4+
5+
CREATE TABLE IF NOT EXISTS pop (
6+
id integer PRIMARY KEY,
7+
name text NOT NULL,
8+
color text NOT NULL
9+
);
10+
11+
GRANT ALL ON DATABASE beverage TO root;

‎db/sql/01-table-insert.sql

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
INSERT INTO "pop" (id, name, color)
2+
VALUES
3+
(1, 'RC Cola', 'brown'),
4+
(2, 'Sprite', 'clear'),
5+
(3, 'Verners', 'brown'),
6+
(4, 'Mt. Lightening', 'green');

‎docker-compose.yml

+173
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
version: "3.4"
2+
3+
services:
4+
5+
py-srv:
6+
build: py-srv
7+
healthcheck:
8+
test: "exit 0"
9+
command: sh -c "/wait && python app.py"
10+
environment:
11+
- WAIT_HOSTS=db:26257,es1:9200,es2:9200,es3:9200
12+
- WAIT_HOSTS_TIMEOUT=300
13+
- WAIT_SLEEP_INTERVAL=30
14+
- WAIT_HOST_CONNECT_TIMEOUT=30
15+
depends_on:
16+
- db
17+
- es1
18+
- es2
19+
- es3
20+
links:
21+
- "db:db"
22+
- "es1:es1"
23+
- "es2:es2"
24+
- "es3:es3"
25+
26+
27+
db:
28+
image: cockroachdb/cockroach:${COCKROACH_VERSION}
29+
container_name: db
30+
ports:
31+
- "20000:26257"
32+
- "8000:8080"
33+
command: start --insecure
34+
35+
db_1:
36+
image: cockroachdb/cockroach:${COCKROACH_VERSION}
37+
container_name: db_1
38+
ports:
39+
- "20001:26257"
40+
- "8001:8080"
41+
command: start --insecure --join=db
42+
43+
db_2:
44+
image: cockroachdb/cockroach:${COCKROACH_VERSION}
45+
container_name: db_2
46+
ports:
47+
- "20002:26257"
48+
- "8002:8080"
49+
command: start --insecure --join=db
50+
51+
db-runner:
52+
image: cockroachdb/cockroach:${COCKROACH_VERSION}
53+
command:
54+
- shell
55+
- /usr/local/bin/init-data.sh
56+
- db:26257
57+
volumes:
58+
- ./db/init-data.sh:/usr/local/bin/init-data.sh:ro
59+
- ./db/sql:/docker-entrypoint-initdb.d
60+
depends_on:
61+
- db
62+
- db_1
63+
- db_2
64+
links:
65+
- "db:db"
66+
67+
68+
es1:
69+
image: elasticsearch:${STACK_VERSION}
70+
container_name: es1
71+
hostname: es1
72+
restart: unless-stopped
73+
healthcheck:
74+
test: "exit 0"
75+
environment:
76+
- "node.store.allow_mmap=false"
77+
- "node.name=es1"
78+
- "bootstrap.memory_lock=true"
79+
- "cluster.name=es-cluster"
80+
- "discovery.seed_hosts=es2,es3"
81+
- "cluster.initial_master_nodes=es1,es2,es3"
82+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m -Des.index.number_of_replicas=0"
83+
- "xpack.security.enabled=false"
84+
- "xpack.security.http.ssl.enabled=false"
85+
- "xpack.security.transport.ssl.enabled=false"
86+
- "xpack.ml.enabled=false"
87+
- "xpack.graph.enabled=false"
88+
- "xpack.watcher.enabled=false"
89+
ulimits:
90+
memlock:
91+
soft: -1
92+
hard: -1
93+
ports:
94+
- 9200:9200
95+
# volumes:
96+
# - ./es/es1/data:/usr/share/elasticsearch/data
97+
# - ./es/es1/log:/usr/share/elasticsearch/log
98+
99+
es2:
100+
image: elasticsearch:${STACK_VERSION}
101+
container_name: es2
102+
hostname: es2
103+
restart: unless-stopped
104+
healthcheck:
105+
test: "exit 0"
106+
environment:
107+
- "node.store.allow_mmap=false"
108+
- "node.name=es2"
109+
- "bootstrap.memory_lock=true"
110+
- "cluster.name=es-cluster"
111+
- "discovery.seed_hosts=es3,es1"
112+
- "cluster.initial_master_nodes=es1,es2,es3"
113+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m -Des.index.number_of_replicas=0"
114+
- "xpack.security.enabled=false"
115+
- "xpack.security.http.ssl.enabled=false"
116+
- "xpack.security.transport.ssl.enabled=false"
117+
- "xpack.ml.enabled=false"
118+
- "xpack.graph.enabled=false"
119+
- "xpack.watcher.enabled=false"
120+
ulimits:
121+
memlock:
122+
soft: -1
123+
hard: -1
124+
ports:
125+
- 9201:9200
126+
# volumes:
127+
# - ./es/es2/data:/usr/share/elasticsearch/data
128+
# - ./es/es2/log:/usr/share/elasticsearch/log
129+
130+
es3:
131+
image: elasticsearch:${STACK_VERSION}
132+
container_name: es3
133+
hostname: es3
134+
restart: unless-stopped
135+
healthcheck:
136+
test: "exit 0"
137+
environment:
138+
- "node.store.allow_mmap=false"
139+
- "node.name=es3"
140+
- "bootstrap.memory_lock=true"
141+
- "cluster.name=es-cluster"
142+
- "discovery.seed_hosts=es2,es1"
143+
- "cluster.initial_master_nodes=es1,es2,es3"
144+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m -Des.index.number_of_replicas=0"
145+
- "xpack.security.enabled=false"
146+
- "xpack.security.http.ssl.enabled=false"
147+
- "xpack.security.transport.ssl.enabled=false"
148+
- "xpack.ml.enabled=false"
149+
- "xpack.graph.enabled=false"
150+
- "xpack.watcher.enabled=false"
151+
ulimits:
152+
memlock:
153+
soft: -1
154+
hard: -1
155+
ports:
156+
- 9202:9200
157+
# volumes:
158+
# - ./es/es3/data:/usr/share/elasticsearch/data
159+
# - ./es/es3/log:/usr/share/elasticsearch/log
160+
161+
kibana:
162+
image: kibana:${STACK_VERSION}
163+
healthcheck:
164+
test: "exit 0"
165+
environment:
166+
- "ELASTICSEARCH_HOSTS=http://es1:9200"
167+
- "SERVER_NAME=127.0.0.1"
168+
ports:
169+
- 5601:5601
170+
depends_on:
171+
- es1
172+
- es2
173+
- es3

‎general.log

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
[2023-03-13 11:15:22 INFO]: install::setup-logging ended
2+
================
3+
[2023-03-13 11:15:22 INFO]: install::start-up started
4+
[2023-03-13 11:15:22 INFO]: install::start-up starting services
5+
[2023-03-13 11:17:45 INFO]: install::root-check started
6+
[2023-03-13 11:17:45 INFO]: install::root-check ended
7+
================
8+
[2023-03-13 11:17:45 INFO]: install::docker-check started
9+
[2023-03-13 11:17:45 INFO]: install::docker-check ended
10+
================
11+
[2023-03-13 11:17:45 INFO]: install::docker-compose-check started
12+
[2023-03-13 11:17:45 INFO]: install::docker-compose-check ended
13+
================
14+
[2023-03-13 11:17:45 INFO]: install::tear-down started
15+
[2023-03-13 11:17:45 INFO]: install::tear-down starting services
16+
[2023-03-13 11:17:45 INFO]: install::tear-down ended
17+
================

0 commit comments

Comments
 (0)