Learning Spark Chapter. 2 스파크 설치 및 무작정 시작하기

• bin
  spark-shell, spark-submit 등 spark 를 실행해 볼 수 있는 실행 파일을 포함한다.
• sbin
  spark process 를 구동(start-all.sh) 하는 파일을 포함한다.
• conf
  spark 설정파일을 포함한다.
  • spark-env.sh
  • spark-default.properties
  • log4j.properties
• R/ec2/python
• example
  간단한 spark 예제들을 포함한다.

log4j.rootCategory=WARN, console

groupId = org.apache.spark artifactId = spark-core_2.10 version = 1.4.0

from pyspark import SparkConf, SparkContext   conf = SparkConf().setMaster(“local”).setAppName(“My App”) sc. SparkContext(conf = conf)

import org.apache.spark.SparkConf import org.apache.spark.SparkContext import org.apache.spark.SparkContext._   var conf = new SparkConf().setMaster(“local”).setAppName(“My App”) var sc = new SparkContext(conf)

import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext;   SparkConf conf = new SparkConf().setMaster(“local”).setAppName(“My App”); JavaSparkContext sc = new JavaSparkContext(conf);

sc.stop(); 또는 System.exit(0); or sys.exit()

package com.tomining.spark.tutorial.example;   import java.io.Serializable; import java.util.Arrays; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFunction; import scala.Tuple2; public class WordCount implements Serializable { public void execute(String inputPath, String outputFile) { SparkConf conf = new SparkConf().setMaster("local").setAppName("WordCount"); JavaSparkContext sc = new JavaSparkContext(conf); JavaRDD<String> input = sc.textFile(inputPath); JavaRDD<String> words = input.flatMap(new FlatMapFunction<String, String>() { public Iterable<String> call(String x) { return Arrays.asList(x.split(" ")); } }); JavaPairRDD<String, Integer> counts = words.mapToPair(new PairFunction<String, String, Integer>() { public Tuple2<String, Integer> call(String x) { return new Tuple2(x, 1); } }).reduceByKey(new Function2<Integer, Integer, Integer>() { public Integer call(Integer x, Integer y) throws Exception { return x + y; } }); counts.saveAsTextFile(outputFile); } }

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">   <modelVersion>4.0.0</modelVersion>   <groupId>com.tomining.spark</groupId>   <artifactId>tutorial</artifactId>   <version>0.0.1-SNAPSHOT</version>   <packaging>jar</packaging>   <name>tutorial</name>   <url>http://maven.apache.org</url>   <properties>     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>   </properties>   <dependencies>     <dependency>       <groupId>junit</groupId>       <artifactId>junit</artifactId>       <version>4.8.2</version>     </dependency>     <dependency>     <groupId>org.apache.spark</groupId>     <artifactId>spark-core_2.10</artifactId>     <version>1.3.0</version>     </dependency>   </dependencies>     <build>   <pluginManagement>   <plugins>   <plugin>   <groupId>org.apache.maven.plugins</groupId>   <artifactId>maven-compiler-plugin</artifactId>   <version>3.1</version>   <configuration>   <source>1.7</source>   <target>1.7</target>   </configuration>   </plugin>   </plugins>   </pluginManagement>   </build> </project>