cdeclog

$ useradd hadoop

$ su - hadoop

$ ssh-keygen -t rsa

$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

$ chmod 0600 ~/.ssh/authorized_keys

$ ssh localhost   # 패스워드 없이 접속이 되어야 함 

wget http://mirror.apache-kr.org/hadoop/common/hadoop-2.7.2/hadoop-2.7.2.tar.gz

tar -zxvf hadoop-2.7.2.tar.gz

sudo cp -R hadoop-2.7.2 /usr/local/

sudo chown -R hadoop:hadoop /usr/local/hadoop-2.7.2

sudo ln -s /usr/local/hadoop-2.7.2 /usr/local/hadoop

# ~/.bashrc 

# HADOOP_HOME  설정 및 bin path 설정 

export HADOOP_HOME=/usr/local/hadoop

export PATH=$PATH:$HADOOP_HOME/bin

# The java implementation to use.

export JAVA_HOME=/usr/local/java

<configuration>

    <property>

        <name>fs.defaultFS</name>

        <value>hdfs://0.0.0.0:9000</value>

    </property>

</configuration>

<configuration>

    <property>

        <name>fs.defaultFS</name>

        <value>hdfs://0.0.0.0:9000</value>

    </property>

    <property>

       <name>hadoop.tmp.dir</name>

       <value>/hdfs/dfs/tmp</value>

    </property>

</configuration>

<configuration>

    <property>

        <name>dfs.replication</name>

        <value>1</value>

    </property>

    <property>

        <name>dfs.namenode.name.dir</name>

        <value>file:/hdfs/dfs/name</value>

    </property>

    <property>

        <name>dfs.datanode.data.dir</name>

        <value>file:/hdfs/dfs/data</value>

    </property>

</configuration>

# dir 경로에 권한 부여 

$ sudo mkdir /hdfs

$ sudo chown hadoop:hadoop /hdfs

# namenode fomat 

$ hdfs namenode -format 

$ cp $HADOOP_HOME/etc/hadoop/mapred-site.xml.template $HADOOP_HOME/etc/hadoop/mapred-site.xml

<configuration>

    <property>

        <name>mapreduce.framework.name</name>

        <value>yarn</value>

    </property>

</configuration>

<configuration>

    <property>

        <name>yarn.nodemanager.aux-services</name>

        <value>mapreduce_shuffle</value>

    </property>

</configuration>

$ sudo - hadoop 

$ $HADOOP_HOME/sbin/start-dfs.sh

$ $HADOOP_HOME/sbin/start-yarn.sh 

# 서비스 확인 

$ jps

2609 DataNode

2485 NameNode

3334 Jps

2779 SecondaryNameNode

2940 ResourceManager

3039 NodeManager

$ hdfs dfs -ls /

$ hdfs dfs -mkdir /data

$ hdfs dfs -put /home/hadoop/data/* /data/

{

    "cmd": ["sbt_bin_path\\sbt.bat", "run"],

    "working_dir": "${project_path:${folder}}",

    "selector": "source.scala"

}

@rem Hadoop home. Required

set HADOOP_HOME=%HADOOP_HOME%

@rem The java implementation to use.  Required.

set JAVA_HOME=%JAVA_HOME%

bin\start-tajo.cmd

> hadoop fs -ls /user/cdecl/data                                                                     

Found 6 items                                                                                        

-rw-r--r--   1 cdecl supergroup       8652 2015-11-13 13:03 /user/cdecl/data/README.txt              

-rw-r--r--   1 cdecl supergroup     569517 2015-11-13 13:03 /user/cdecl/data/links.csv               

-rw-r--r--   1 cdecl supergroup    1397542 2015-11-13 13:03 /user/cdecl/data/movies.csv              

-rw-r--r--   1 cdecl supergroup        258 2015-11-13 13:03 /user/cdecl/data/movies.csv.dsn          

-rw-r--r--   1 cdecl supergroup  533444411 2015-11-13 13:03 /user/cdecl/data/ratings.csv             

-rw-r--r--   1 cdecl supergroup   16603996 2015-11-13 13:03 /user/cdecl/data/tags.csv              

- ratings.csv 

- 영화 평점 정보, 약 500MB, 20,000,264 rows

- movies.csv

- 영화 정보, 약 1MB , 27,279 rows

D:\hadoop\tajo-0.11.0

> bin\tsql

starting cli, logging to D:\hadoop\tajo-0.11.0\logs\tajo.log

Try \? for help.

default>

CREATE EXTERNAL table movies ( mid int,  title text,  genres text )

USING TEXT WITH ('text.delimiter'=',', 'text.skip.headerlines'='1')

LOCATION 'hdfs://localhost:9000/user/cdecl/data/movies.csv';

create EXTERNAL table ratings ( userid int, mid int, rate int, timest text )

USING TEXT WITH ('text.delimiter'=',', 'text.skip.headerlines'='1')

LOCATION 'hdfs://localhost:9000/user/cdecl/data/ratings.csv';

SELECT a.mid, max(b.title), avg(a.rate) 

FROM ratings a join movies b on a.mid = b.mid 

GROUP BY a.mid 

ORDER BY avg(a.rate) DESC 

LIMIT 10;

* 시스템-고급 시스템 설정-환경변수 세팅 

HADOOP_HOME=D:\hadoop\hadoop-2.7.1

JAVA_HOME=D:\hadoop\Java

PATH=%PATH%;D:\hadoop\hadoop-2.7.1\bin

* Java를 기본 설치 할 경우 공백을 들어간 "C:\Program Files\Java\jre1.8.0_66"에 설치가 되는데 mklink를 이용하여 공백없는 경로에 심볼릭 링크 생성 

mklink /j d:\hadoop\Java "C:\Program Files\Java\jre1.8.0_66"

d:\hadoop\Java <<===>> C:\Program Files\Java\jre1.8.0_66에 대한 교차점을 만들었습니다.

fs.defaultFS:

The name of the default file system. A URI whose scheme and authority determine the FileSystem implementation. The uri's scheme determines the config property (fs.SCHEME.impl) naming the FileSystem implementation class. The uri's authority is used to determine the host, port, etc. for a filesystem.

dfs.replication:

Default block replication. The actual number of replications can be specified when the file is created. The default is used if replication is not specified in create time.

dfs.namenode.name.dir:

Determines where on the local filesystem the DFS name node should store the name table(fsimage). If this is a comma-delimited list of directories then the name table is replicated in all of the directories, for redundancy.

dfs.datanode.data.dir:

Determines where on the local filesystem an DFS data node should store its blocks. If this is a comma-delimited list of directories, then data will be stored in all named directories, typically on different devices. Directories that do not exist are ignored.

yarn.nodemanager.aux-services:

The auxiliary service name. Default value is omapreduce_shuffle

yarn.nodemanager.aux-services.mapreduce.shuffle.class:

The auxiliary service class to use. Default value is org.apache.hadoop.mapred.ShuffleHandler

yarn.application.classpath:

CLASSPATH for YARN applications. A comma-separated list of CLASSPATH entries.

mapreduce.framework.name:

The runtime framework for executing MapReduce jobs. Can be one of local, classic or yarn.

%HADOOP_HOME%\bin\hdfs namenode -format 

%HADOOP_HOME%sbin\start-dfs.cmd

%HADOOP_HOME%sbin\start-yarn.cmd

* 콘솔창이 4개가 실행되면서 각각의 서비스 시작 

- http://localhost:8042 - Resource Manager and Node Manager : 

- http://localhost:50070 - Namenode 

* http://localhost:9000 - Service URI

> hadoop fs -mkdir -p /user/cdecl/data 

> hadoop fs -ls /                                                                                    

Found 1 items                                                                                        

drwxr-xr-x   - cdecl supergroup          0 2015-11-13 13:02 /user

> hadoop fs -put D:\hadoop\data\ml-20m\* /user/cdecl/data

> hadoop fs -ls /user/cdecl/data                                                                     

Found 6 items                                                                                        

-rw-r--r--   1 cdecl supergroup       8652 2015-11-13 13:03 /user/cdecl/data/README.txt              

-rw-r--r--   1 cdecl supergroup     569517 2015-11-13 13:03 /user/cdecl/data/links.csv               

-rw-r--r--   1 cdecl supergroup    1397542 2015-11-13 13:03 /user/cdecl/data/movies.csv              

-rw-r--r--   1 cdecl supergroup        258 2015-11-13 13:03 /user/cdecl/data/movies.csv.dsn          

-rw-r--r--   1 cdecl supergroup  533444411 2015-11-13 13:03 /user/cdecl/data/ratings.csv             

-rw-r--r--   1 cdecl supergroup   16603996 2015-11-13 13:03 /user/cdecl/data/tags.csv                

# SparkApp
from pyspark import SparkContext


def main():
	sc = SparkContext("local[*]", "SparkApp")

	ratings = "hdfs://localhost:9000/user/cdecl/data/movies.csv"
	rdd = sc.textFile(ratings)

	print(rdd.take(5))	


if __name__ == "__main__":
	main()

['movieId,title,genres', '1,Toy Story (1995),Adventure|Animation|Children|Comedy|Fantasy', '2,Jumanji (1995),Adventure|Children|Fantasy', '3,Grumpier Old Men (1995),Comedy|Romance', '4,Waiting to Exhale (1995),Comedy|Drama|Romance']

%SPARK_HOME%\bin\spark-submit test.py

98

python test.py

98

Ratings Data File Structure (ratings.csv)

-----------------------------------------

All ratings are contained in the file `ratings.csv`.

    userId,movieId,rating,timestamp

userId,movieId,rating,timestamp

138493,60816,4.5,1259865163

138493,61160,4.0,1258390537

138493,65682,4.5,1255816373

138493,66762,4.5,1255805408

138493,68319,4.5,1260209720

Movies Data File Structure (movies.csv)

---------------------------------------

Movie information is contained in the file `movies.csv`. Each line of this file after the header row represents one movie, and has the following format:

    movieId,title,genres

movieId,title,genres

131241,Ants in the Pants (2000),Comedy|Romance

131243,Werner - Gekotzt wird später (2003),Animation|Comedy

131248,Brother Bear 2 (2006),Adventure|Animation|Children|Comedy|Fantasy

131250,No More School (2000),Comedy

131252,Forklift Driver Klaus: The First Day on the Job (2001),Comedy|Horror

# spark_movelens.py

from pyspark import SparkContext
from statistics import mean


if __name__ == "__main__":
	sc = SparkContext("local[*]", "Simple App")

	ratings = "D:/hadoop/data/ml-20m/ratings.csv"
	movies = "D:/hadoop/data/ml-20m/movies.csv"

	movie = sc.textFile(movies)
	header = movie.first()
	dic = movie.filter(lambda x: x != header).map(lambda x: x.split(',')).map(lambda x: (x[0],x[1]))

	data = sc.textFile(ratings)
	header = data.first()
	rdd = data.filter(lambda x: x != header).map(lambda x: x.split(','))\
		.map(lambda x: (x[1], float(x[2]))).groupByKey().mapValues(list)\
		.map(lambda x: (x[0], round(mean(x[1]), 2)))\
		.join(dic).sortBy(lambda x: x[1][0])\
		.map(lambda x: (x[0], x[1][0], x[1][1]))
		
	for id, avg, title in rdd.collect():
	 	print('{} {} - {}'.format(id, avg, title.encode('utf8')))

# Ubuntu 

sudo apt-get install python-software-properties

sudo add-apt-repository ppa:webupd8team/java

sudo apt-get update

sudo apt-get install oracle-java8-installer

# Windows 

http://www.java.com/ko/download/

http://spark.apache.org/downloads.html (Spark 1.5.2 Pre-built for Hadoop 2.6 and later)

# Ubuntu 

wget http://apache.mirror.cdnetworks.com/spark/spark-1.5.2/spark-1.5.2-bin-hadoop2.6.tgz

tar -zxvf spark-1.5.2/spark-1.5.2-bin-hadoop2.6.tgz

sudo mv spark-1.5.2-bin-hadoop2.6 /usr/local/

sudo ln -s /usr/local/spark-1.5.2-bin-hadoop2.6 /usr/local/spark

https://www.python.org/

* Ubuntu 15.10 의 경우 3.4 버전까지 기본 설치 되어 있음

# Hadoop for Windows 

비공식 바이너리 : https://github.com/karthikj1/Hadoop-2.7.1-Windows-64-binaries/releases 

# Ubuntu 예제 (.profile)

export JAVA_HOME=/usr/lib/jvm/java-8-oracle

export SPARK_HOME=/usr/local/spark

export PYTHONPATH=$SPARK_HOME/python/lib/pyspark.zip:$SPARK_HOME/python/lib/py4j-0.8.2.1-src.zip

export HADOOP_HOME=/usr/local/hadoop   # 옵션

# Windows 환경변수 세팅 (공백이 없는 경로로 하며, 필요시 mklink를 이용하여 심볼릭 링크 설정을 하면 편함)

%JAVA_HOME%

%SPARK_HOME%

%PYTHONPATH%  

%HADOOP_HOME%

* Windows의 경우 %HADOOP_HOME% 세팅을 하지 않는 경우 아래와 같은 에러가 발생 

ERROR Shell: Failed to locate the winutils binary in the hadoop binary path

java.io.IOException: Could not locate executable null\bin\winutils.exe in the Hadoop binaries.

* Windows 에서 hive 에러 날 경우 

%HADOOP_HOME%/bin/winutils.exe chmod 777 D:\tmp\hive

cp $SPARK_HOME/conf/log4j.properties.template log4j.properties

# log4j.properties 파일편집

log4j.rootCategory=WARN, console  # INFO->WARN, INFO정보가 많이 출력 되므로 수정