Skip to content

Commit 94a65a5

Browse files
binary-joesjmillington
binary-joe
authored and
sjmillington
committed
BAEL-398 introduction to genie (eugenp#8217)
1 parent 10be32c commit 94a65a5

File tree

7 files changed

+295
-0
lines changed

7 files changed

+295
-0
lines changed

‎netflix/README.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
## Netflix
2+
3+
This module contains articles about Netflix.
4+
5+
### Relevant articles
6+
7+
- [Introduction to Netflix Genie](https://github.com/eugenp/tutorials/tree/master/netflix/genie)

‎netflix/genie/pom.xml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
5+
<groupId>com.baeldung</groupId>
6+
<artifactId>genie</artifactId>
7+
<packaging>jar</packaging>
8+
<version>1.0.0-SNAPSHOT</version>
9+
10+
<name>Genie</name>
11+
<description>Sample project for Netflix Genie</description>
12+
13+
<parent>
14+
<groupId>com.baeldung</groupId>
15+
<artifactId>netflix</artifactId>
16+
<version>1.0.0-SNAPSHOT</version>
17+
</parent>
18+
</project>

‎netflix/genie/src/main/resources/com/baeldung/netflix/genie/docker/docker-compose.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
version: "2"
2+
services:
3+
genie:
4+
image: netflixoss/genie-app:3.3.9
5+
ports:
6+
- "8080:8080"
7+
depends_on:
8+
- genie-hadoop-prod
9+
- genie-hadoop-test
10+
- genie-apache
11+
tty: true
12+
container_name: genie_demo_app_3.3.9
13+
genie-apache:
14+
image: netflixoss/genie-demo-apache:3.3.9
15+
tty: true
16+
container_name: genie_demo_apache_3.3.9
17+
genie-client:
18+
image: netflixoss/genie-demo-client:3.3.9
19+
depends_on:
20+
- genie
21+
tty: true
22+
container_name: genie_demo_client_3.3.9
23+
genie-hadoop-prod:
24+
image: sequenceiq/hadoop-docker:2.7.1
25+
command: /bin/bash -c "/usr/local/hadoop/sbin/mr-jobhistory-daemon.sh start historyserver && /etc/bootstrap.sh -bash"
26+
ports:
27+
- "19888:19888"
28+
- "19070:50070"
29+
- "19075:50075"
30+
- "8088:8088"
31+
tty: true
32+
container_name: genie_demo_hadoop_prod_3.3.9
33+
genie-hadoop-test:
34+
image: sequenceiq/hadoop-docker:2.7.1
35+
command: /bin/bash -c "/usr/local/hadoop/sbin/mr-jobhistory-daemon.sh start historyserver && /etc/bootstrap.sh -bash"
36+
ports:
37+
- "19889:19888"
38+
- "19071:50070"
39+
- "19076:50075"
40+
- "8089:8088"
41+
tty: true
42+
container_name: genie_demo_hadoop_test_3.3.9

‎netflix/genie/src/main/resources/com/baeldung/netflix/genie/docker/init_demo.py

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
#!/usr/bin/python2.7
2+
3+
# Copyright 2016 Netflix, Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import logging
18+
19+
import yaml
20+
from pygenie.client import Genie
21+
from pygenie.conf import GenieConf
22+
23+
logging.basicConfig(level=logging.WARNING)
24+
25+
LOGGER = logging.getLogger(__name__)
26+
27+
28+
def load_yaml(yaml_file):
29+
with open(yaml_file) as _file:
30+
return yaml.load(_file)
31+
32+
33+
genie_conf = GenieConf()
34+
genie_conf.genie.url = "http://genie:8080"
35+
36+
genie = Genie(genie_conf)
37+
38+
hadoop_application = load_yaml("applications/hadoop271.yml")
39+
hadoop_application_id = genie.create_application(hadoop_application)
40+
LOGGER.warn("Created Hadoop 2.7.1 application with id = [%s]" % hadoop_application_id)
41+
42+
spark_163_application = load_yaml("applications/spark163.yml")
43+
spark_163_application_id = genie.create_application(spark_163_application)
44+
LOGGER.warn("Created Spark 1.6.3 application with id = [%s]" % spark_163_application_id)
45+
46+
spark_201_application = load_yaml("applications/spark201.yml")
47+
spark_201_application_id = genie.create_application(spark_201_application)
48+
LOGGER.warn("Created Spark 2.0.1 application with id = [%s]" % spark_201_application_id)
49+
50+
hadoop_command = load_yaml("commands/hadoop271.yml")
51+
hadoop_command_id = genie.create_command(hadoop_command)
52+
LOGGER.warn("Created Hadoop command with id = [%s]" % hadoop_command_id)
53+
54+
hdfs_command = load_yaml("commands/hdfs271.yml")
55+
hdfs_command_id = genie.create_command(hdfs_command)
56+
LOGGER.warn("Created HDFS command with id = [%s]" % hdfs_command_id)
57+
58+
yarn_command = load_yaml("commands/yarn271.yml")
59+
yarn_command_id = genie.create_command(yarn_command)
60+
LOGGER.warn("Created Yarn command with id = [%s]" % yarn_command_id)
61+
62+
spark_163_shell_command = load_yaml("commands/sparkShell163.yml")
63+
spark_163_shell_command_id = genie.create_command(spark_163_shell_command)
64+
LOGGER.warn("Created Spark 1.6.3 Shell command with id = [%s]" % spark_163_shell_command_id)
65+
66+
spark_163_submit_command = load_yaml("commands/sparkSubmit163.yml")
67+
spark_163_submit_command_id = genie.create_command(spark_163_submit_command)
68+
LOGGER.warn("Created Spark 1.6.3 Submit command with id = [%s]" % spark_163_submit_command_id)
69+
70+
spark_201_shell_command = load_yaml("commands/sparkShell201.yml")
71+
spark_201_shell_command_id = genie.create_command(spark_201_shell_command)
72+
LOGGER.warn("Created Spark 2.0.1 Shell command with id = [%s]" % spark_201_shell_command_id)
73+
74+
spark_201_submit_command = load_yaml("commands/sparkSubmit201.yml")
75+
spark_201_submit_command_id = genie.create_command(spark_201_submit_command)
76+
LOGGER.warn("Created Spark 2.0.1 Submit command with id = [%s]" % spark_201_submit_command_id)
77+
78+
genie.set_application_for_command(hadoop_command_id, [hadoop_application_id])
79+
LOGGER.warn("Set applications for Hadoop command to = [%s]" % hadoop_application_id)
80+
81+
genie.set_application_for_command(hdfs_command_id, [hadoop_application_id])
82+
LOGGER.warn("Set applications for HDFS command to = [[%s]]" % hadoop_application_id)
83+
84+
genie.set_application_for_command(yarn_command_id, [hadoop_application_id])
85+
LOGGER.warn("Set applications for Yarn command to = [[%s]]" % hadoop_application_id)
86+
87+
genie.set_application_for_command(spark_163_shell_command_id, [hadoop_application_id, spark_163_application_id])
88+
LOGGER.warn("Set applications for Spark 1.6.3 Shell command to = [%s]" %
89+
[hadoop_application_id, spark_163_application_id])
90+
91+
genie.set_application_for_command(spark_163_submit_command_id, [hadoop_application_id, spark_163_application_id])
92+
LOGGER.warn("Set applications for Spark 1.6.3 Submit command to = [%s]" %
93+
[hadoop_application_id, spark_163_application_id])
94+
95+
genie.set_application_for_command(spark_201_shell_command_id, [hadoop_application_id, spark_201_application_id])
96+
LOGGER.warn("Set applications for Spark 2.0.1 Shell command to = [%s]" %
97+
[hadoop_application_id, spark_201_application_id])
98+
99+
genie.set_application_for_command(spark_201_submit_command_id, [hadoop_application_id, spark_201_application_id])
100+
LOGGER.warn("Set applications for Spark 2.0.1 Submit command to = [%s]" %
101+
[hadoop_application_id, spark_201_application_id])
102+
103+
prod_cluster = load_yaml("clusters/prod.yml")
104+
prod_cluster_id = genie.create_cluster(prod_cluster)
105+
LOGGER.warn("Created prod cluster with id = [%s]" % prod_cluster_id)
106+
107+
test_cluster = load_yaml("clusters/test.yml")
108+
test_cluster_id = genie.create_cluster(test_cluster)
109+
LOGGER.warn("Created test cluster with id = [%s]" % test_cluster_id)
110+
111+
genie.set_commands_for_cluster(
112+
prod_cluster_id,
113+
[hadoop_command_id, hdfs_command_id, yarn_command_id, spark_163_shell_command_id, spark_201_shell_command_id,
114+
spark_163_submit_command_id, spark_201_submit_command_id]
115+
)
116+
LOGGER.warn("Added all commands to the prod cluster with id = [%s]" % prod_cluster_id)
117+
genie.set_commands_for_cluster(
118+
test_cluster_id,
119+
[hadoop_command_id, hdfs_command_id, yarn_command_id, spark_163_shell_command_id, spark_201_shell_command_id,
120+
spark_163_submit_command_id, spark_201_submit_command_id]
121+
)
122+
LOGGER.warn("Added all commands to the test cluster with id = [%s]" % test_cluster_id)

‎netflix/genie/src/main/resources/com/baeldung/netflix/genie/docker/run_spark_submit_job.py

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# Copyright 2016 Netflix, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
##################################################################################
16+
# This script assumes setup.py has already been run to configure Genie and that
17+
# this script is executed on the host where Genie is running. If it's executed on
18+
# another host change the localhost line below.
19+
##################################################################################
20+
21+
from __future__ import absolute_import, division, print_function, unicode_literals
22+
23+
import logging
24+
import sys
25+
26+
import pygenie
27+
28+
logging.basicConfig(level=logging.ERROR)
29+
30+
LOGGER = logging.getLogger(__name__)
31+
32+
pygenie.conf.DEFAULT_GENIE_URL = "http://genie:8080"
33+
34+
# Create a job instance and fill in the required parameters
35+
job = pygenie.jobs.GenieJob() \
36+
.genie_username('root') \
37+
.job_version('3.0.0')
38+
39+
# Set cluster criteria which determine the cluster to run the job on
40+
job.cluster_tags(['sched:' + str(sys.argv[1]), 'type:yarn'])
41+
42+
# Set command criteria which will determine what command Genie executes for the job
43+
if len(sys.argv) == 2:
44+
# Use the default spark
45+
job.command_tags(['type:spark-submit'])
46+
job.job_name('Genie Demo Spark Submit Job')
47+
else:
48+
# Use the spark version passed in
49+
job.command_tags(['type:spark-submit', 'ver:' + str(sys.argv[2])])
50+
job.job_name('Genie Demo Spark ' + str(sys.argv[2]) + ' Submit Job')
51+
52+
# Any command line arguments to run along with the command. In this case it holds
53+
# the actual query but this could also be done via an attachment or file dependency.
54+
# This jar ___location is where it is installed on the Genie node but could also pass
55+
# the jar as attachment and use it locally
56+
if len(sys.argv) == 2:
57+
# Default is spark 1.6.3
58+
job.command_arguments(
59+
"--class org.apache.spark.examples.SparkPi "
60+
"${SPARK_HOME}/lib/spark-examples*.jar "
61+
"10"
62+
)
63+
else:
64+
# Override with Spark 2.x ___location
65+
job.command_arguments(
66+
"--class org.apache.spark.examples.SparkPi "
67+
"${SPARK_HOME}/examples/jars/spark-examples*.jar "
68+
"10"
69+
)
70+
71+
# Submit the job to Genie
72+
running_job = job.execute()
73+
74+
print('Job {} is {}'.format(running_job.job_id, running_job.status))
75+
print(running_job.job_link)
76+
77+
# Block and wait until job is done
78+
running_job.wait()
79+
80+
print('Job {} finished with status {}'.format(running_job.job_id, running_job.status))

‎netflix/pom.xml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
5+
<groupId>com.baeldung</groupId>
6+
<artifactId>netflix</artifactId>
7+
<packaging>pom</packaging>
8+
<version>1.0.0-SNAPSHOT</version>
9+
10+
<name>Netflix</name>
11+
<description>Module for Netflix projects</description>
12+
13+
<parent>
14+
<groupId>com.baeldung</groupId>
15+
<artifactId>parent-modules</artifactId>
16+
<version>1.0.0-SNAPSHOT</version>
17+
</parent>
18+
19+
<modules>
20+
<module>genie</module>
21+
</modules>
22+
</project>

‎pom.xml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,7 @@
590590
<module>mustache</module>
591591
<module>mybatis</module>
592592

593+
<module>netflix</module>
593594

594595
<module>optaplanner</module>
595596
<module>orika</module>
@@ -669,6 +670,8 @@
669670
</build>
670671

671672
<modules>
673+
<module>netflix</module>
674+
672675
<module>parent-boot-1</module>
673676
<module>parent-boot-2</module>
674677
<module>parent-spring-4</module>
@@ -1220,6 +1223,7 @@
12201223
<module>mustache</module>
12211224
<module>mybatis</module>
12221225

1226+
<module>netflix</module>
12231227

12241228
<module>optaplanner</module>
12251229
<module>orika</module>

0 commit comments

Comments
 (0)