Skip to content

Commit

Permalink
Set hadoop host from environment variable in config files
Browse files Browse the repository at this point in the history
  • Loading branch information
hfxbse committed Nov 18, 2024
1 parent c792242 commit 302837f
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 0 deletions.
5 changes: 5 additions & 0 deletions airflow.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,18 @@ FROM marcelmittelstaedt/airflow:latest

LABEL org.opencontainers.image.source=https://github.com/hfxbse/dhbw-big-data

ENV HADOOP_HOST="hadoop"

RUN sed -i '34,41d' /startup.sh
RUN sed -i '3 i service ssh start' /startup.sh
RUN git clone --depth 1 https://github.com/marcelmittelstaedt/BigData.git /tmp/upstream
RUN wget https://jdbc.postgresql.org/download/postgresql-42.7.4.jar -P /home/airflow/spark/jars/
RUN mv /tmp/upstream/exercises/winter_semester_2024-2025/05_airflow/plugins /home/airflow/airflow
RUN mv /tmp/upstream/exercises/winter_semester_2024-2025/05_airflow/dags /home/airflow/airflow
RUN mv /tmp/upstream/exercises/winter_semester_2024-2025/05_airflow/python /home/airflow/airflow
RUN sed -i 's/hadoop:/${HADOOP_HOST}:/g' /home/airflow/hadoop/etc/hadoop/core-site.xml
RUN sed -i 's/hadoop:/${HADOOP_HOST}:/g' /home/airflow/hadoop/etc/hadoop/yarn-site.xml

COPY airflow/ /home/airflow/airflow/
COPY spark/ /home/airflow/airflow/python/
RUN chown -R airflow /home/airflow/airflow
Expand Down
3 changes: 3 additions & 0 deletions hadoop.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@ FROM marcelmittelstaedt/spark_base:latest

LABEL org.opencontainers.image.source=https://github.com/hfxbse/dhbw-big-data

ENV HADOOP_HOST="hadoop"

RUN sed -i '44,50 s/^#//' /startup.sh
RUN head -n -9 /startup.sh > temp.sh ; mv temp.sh /startup.sh
RUN sed -i 's/hadoop:9000/${HADOOP_HOST}:9000/' /home/hadoop/hadoop/etc/hadoop/core-site.xml
RUN echo "echo executing hiveserver2; sudo -u hadoop -H sh -c /home/hadoop/hive/bin/hiveserver2" >> /startup.sh
RUN chmod +x /startup.sh

Expand Down

0 comments on commit 302837f

Please sign in to comment.