Hoptimator/Makefile at main · linkedin/Hoptimator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156

install:
	./gradlew compileJava installDist

test:
	./gradlew test -x spotbugsMain -x spotbugsTest -x spotbugsTestFixtures

coverage:
	./gradlew jacocoAggregateReport -x spotbugsMain -x spotbugsTest -x spotbugsTestFixtures
	@echo "Aggregate report: file://$(shell pwd)/build/reports/jacoco/aggregate/index.html"

build:
	./gradlew build jacocoAggregateReport shadowJar
	docker build . -t hoptimator
	docker build hoptimator-flink-runner -f hoptimator-flink-runner/Dockerfile-flink-runner -t hoptimator-flink-runner
	docker build hoptimator-flink-runner -f hoptimator-flink-runner/Dockerfile-flink-operator -t hoptimator-flink-operator

bounce: build undeploy deploy

clean:
	./gradlew clean

deploy-config:
	kubectl apply -f ./deploy/config/hoptimator-configmap.yaml

undeploy-config:
	kubectl delete configmap hoptimator-configmap || echo "skipping"

deploy: deploy-config
	kubectl apply -f ./hoptimator-k8s/src/main/resources/
	kubectl apply -f ./deploy
	kubectl apply -f ./deploy/dev/rbac.yaml
	kubectl wait --for=condition=Established=True	\
	  crds/subscriptions.hoptimator.linkedin.com \
	  crds/kafkatopics.hoptimator.linkedin.com \
	  crds/sqljobs.hoptimator.linkedin.com

undeploy: undeploy-config
	kubectl delete -f ./deploy/dev/rbac.yaml || echo "skipping"
	kubectl delete -f ./deploy || echo "skipping"
	kubectl delete -f ./hoptimator-k8s/src/main/resources/ || echo "skipping"

quickstart: build deploy

deploy-demo: deploy
	kubectl apply -f ./deploy/samples/demodb.yaml
	kubectl apply -f ./deploy/samples/tabletriggers.yaml
	kubectl apply -f ./deploy/samples/retl-job-template.yaml
	kubectl apply -f ./deploy/samples/crontrigger.yaml
	kubectl apply -f ./deploy/samples/user-jobs.yaml

undeploy-demo: undeploy
	kubectl delete -f ./deploy/samples/demodb.yaml || echo "skipping"
	kubectl delete -f ./deploy/samples/tabletriggers.yaml || echo "skipping"
	kubectl delete -f ./deploy/samples/retl-job-template.yaml || echo "skipping"
	kubectl delete -f ./deploy/samples/crontrigger.yaml || echo "skipping"

deploy-flink: deploy
	kubectl create namespace flink || echo "skipping"
	kubectl create -f https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml || echo "skipping"
	helm repo add flink-operator-repo https://downloads.apache.org/flink/flink-kubernetes-operator-1.13.0/
	helm upgrade --install --atomic --set webhook.create=false,image.pullPolicy=Never,image.repository=docker.io/library/hoptimator-flink-operator,image.tag=latest --set-json='watchNamespaces=["default","flink"]' flink-kubernetes-operator flink-operator-repo/flink-kubernetes-operator
	kubectl apply -f ./deploy/dev/flink-session-cluster.yaml
	kubectl apply -f ./deploy/dev/flink-sql-gateway.yaml
	kubectl apply -f ./deploy/samples/flink-template.yaml
	kubectl apply -f ./deploy/samples/flink-beam-template.yaml

undeploy-flink:
	kubectl delete flinksessionjobs.flink.apache.org --all || echo "skipping"
	kubectl delete flinkdeployments.flink.apache.org --all || echo "skipping"
	kubectl delete crd flinksessionjobs.flink.apache.org || echo "skipping"
	kubectl delete crd flinkdeployments.flink.apache.org || echo "skipping"
	helm uninstall flink-kubernetes-operator || echo "skipping"
	helm repo remove flink-operator-repo || echo "skipping"
	kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml || echo "skipping"

deploy-kafka: deploy deploy-flink
	kubectl create namespace kafka || echo "skipping"
	kubectl apply -f "https://strimzi.io/install/latest?namespace=kafka" -n kafka
	sleep 10 # avoid kubectl race condition
	kubectl wait --for=condition=Established=True crds/kafkas.kafka.strimzi.io
	kubectl apply -f ./deploy/samples/kafkadb.yaml
	kubectl apply -f ./deploy/dev/kafka.yaml
	kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=10m -n kafka
	kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-1 --for=condition=Ready --timeout=10m
	kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-2 --for=condition=Ready --timeout=10m

undeploy-kafka:
	kubectl delete kafkatopic.kafka.strimzi.io --all || echo "skipping"
	kubectl delete strimzi -n kafka --all || echo "skipping"
	kubectl delete pvc -l strimzi.io/name=one-kafka -n kafka || echo "skipping"
	kubectl delete -f "https://strimzi.io/install/latest?namespace=kafka" -n kafka || echo "skipping"
	kubectl delete -f ./deploy/samples/kafkadb.yaml || echo "skipping"
	kubectl delete namespace kafka || echo "skipping"

# Deploys MySQL cluster in docker with a few databases and tables.
deploy-mysql: deploy deploy-flink
	docker compose -f ./deploy/docker/mysql/docker-compose.yaml up -d --wait
	kubectl apply -f ./deploy/samples/mysqldb.yaml

undeploy-mysql:
	kubectl delete -f ./deploy/samples/mysqldb.yaml || echo "skipping"
	docker compose -f ./deploy/docker/mysql/docker-compose.yaml down

# Deploys Venice cluster in docker and creates two stores in Venice. Stores are not managed via K8s for now.
deploy-venice: deploy deploy-flink
	docker compose -f ./deploy/docker/venice/docker-compose-single-dc-setup.yaml up -d --wait
	docker exec venice-client ./create-store.sh http://venice-controller:5555 venice-cluster0 test-store schemas/keySchemaRecord.avsc schemas/valueSchema.avsc
	docker exec venice-client ./create-store.sh http://venice-controller:5555 venice-cluster0 test-store-primitive schemas/keySchemaPrimitive.avsc schemas/valueSchema.avsc
	kubectl apply -f ./deploy/samples/venicedb.yaml

undeploy-venice:
	kubectl delete -f ./deploy/samples/venicedb.yaml || echo "skipping"
	docker compose -f ./deploy/docker/venice/docker-compose-single-dc-setup.yaml down

deploy-dev-environment: deploy deploy-demo deploy-flink deploy-kafka deploy-mysql deploy-venice

undeploy-dev-environment: undeploy-venice undeploy-mysql undeploy-kafka undeploy-flink undeploy-demo undeploy
	kubectl delete -f ./deploy/dev || echo "skipping"

# Integration test setup intended to be run locally
integration-tests: deploy-dev-environment
	kubectl port-forward -n kafka svc/one-kafka-external-bootstrap 9092 & echo $$! > port-forward.pid
	kubectl port-forward -n flink svc/flink-sql-gateway 8083 & echo $$! > port-forward-2.pid
	kubectl port-forward -n flink svc/basic-session-deployment-rest 8081 & echo $$! > port-forward-3.pid
	./gradlew intTest --no-parallel || kill `cat port-forward.pid port-forward-2.pid port-forward-3.pid`
	kill `cat port-forward.pid`
	kill `cat port-forward-2.pid`
	kill `cat port-forward-3.pid`

# kind cluster used in github workflow needs to have different routing set up, avoiding the need to forward kafka ports
integration-tests-kind: deploy-dev-environment
	./gradlew intTest -i --no-parallel

generate-models:
	./generate-models.sh
	./hoptimator-models/generate-models.sh # <-- marked for deletion

release:
	test -n "$(VERSION)"  # MISSING ARG: $$VERSION
	./gradlew publish

build-zeppelin: build
	docker build -t hoptimator-zeppelin -t hoptimator-zeppelin:0.11.2 -f ./deploy/docker/zeppelin/Dockerfile-zeppelin .

# attaches to terminal (not run as daemon)
run-zeppelin: build-zeppelin
	kubectl apply -f deploy/docker/zeppelin/zeppelin-flink-engine.yaml
	kubectl apply -f deploy/docker/zeppelin/zeppelin-kafkadb.yaml
	docker run --rm -p 8080:8080 \
	  --volume=${HOME}/.kube/config:/opt/zeppelin/.kube/config \
	  --add-host=docker-for-desktop:host-gateway \
	  --name hoptimator-zeppelin \
	  hoptimator-zeppelin

.PHONY: install test coverage build bounce clean quickstart deploy-config undeploy-config deploy undeploy deploy-demo undeploy-demo deploy-flink undeploy-flink deploy-kafka undeploy-kafka deploy-mysql undeploy-mysql deploy-venice undeploy-venice build-zeppelin run-zeppelin integration-tests integration-tests-kind deploy-dev-environment undeploy-dev-environment generate-models release