[HDFS] Hadoop 클러스터 구축

0. prerequisite

컴퓨터 총 세대를 준비하였다.

name node * 1

data node * 2

	ubuntu	hadoop	java
version	20.04	3.3.1	11

1. hadoop source code 다운로드

wget https://dlcdn.apache.org/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz
tar xzf hadoop-3.3.1.tar.gz

2. /etc/hosts 수정

sudo vim /etc/hosts

10.12.168.200 hadoop-master-00
10.12.168.201 hadoop-slave-00
10.12.168.202 hadoop-slave-01

/etc/hosts

3. ssh key 생성 및 복제

ssh-keygen -t rsa
ssh-copy-id ubuntu@hadoop-master-00
ssh-copy-id ubuntu@hadoop-slave-00
ssh-copy-id ubuntu@hadoop-slave-01

4. HADOOP 환경 변수 등록 (~/.bashrc 혹은 ~/.zshrc)

#Hadoop Related Options
export HADOOP_HOME=/home/hdoop/hadoop-3.3.1
export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME

~/.bashrc

5. java 설치 및 java home 설정

# if java is not installed
sudo apt-get update && sudo apt-get upgrade
sudo apt-get install openjdk-11-jdk -y

readlink -f /usr/bin/javac

마찬가지로 ~/.bashrc 또는 ~/.zshrc 에서,

export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64

6. hadoop 설정 변경

---

$HADOOP_HOME/etc/hadoop/core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/home/hdoop/tmpdata</value>
        </property>
        <property>
                <name>fs.default.name</name>
                <value>hdfs://hadoop-master:9000</value>
        </property>
</configuration>

---

$HADOOP_HOME/etc/hadoop/mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
</configuration>

---

$HADOOP_HOME/etc/hadoop/hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
        <property>
                <name>dfs.data.dir</name>
                <value>/home/hdoop/dfsdata/namenode</value>
        </property>
        <property>
                <name>dfs.data.dir</name>
                <value>/home/hdoop/dfsdata/datanode</value>
        </property>
        <property>
                <name>dfs.replication</name>
                <value>2</value>
        </property>
</configuration>

---

$HADOOP_HOME/etc/hadoop/workers

hadoop-slave-00
hadoop-slave-01

7. hadoop 시작

$HADOOP_HOME/sbin/start-all.sh