리눅스 관련 게시판
2015/07/03(22:56) from 58.125.30.140
작성자 : 주인장 조회수 : 5743 , 줄수 : 225
Re: [Hadoop] 2. wordcount 예제
[root@hadoop01 hdfs]# ls -l
total 16
drwxrwxr-x. 2 hadoop hadoop 4096 Jul  3 09:09 data
drwxr-xr-x. 6 root   root   4096 Jul  3 23:19 mapred
drwxrwxr-x. 5 hadoop hadoop 4096 Jul  3 23:19 name
drwxrwxr-x. 2 hadoop hadoop 4096 Jul  3 09:09 temp

[root@hadoop01 hdfs]# hadoop fs -ls
ls: Cannot access .: No such file or directory.

[root@hadoop01 hdfs]# hadoop fs -mkdir /input

[root@hadoop01 hadoop]# pwd
/usr/local/hadoop
[root@hadoop01 hadoop]# ls
bin          contrib                  hadoop-examples-1.2.1.jar     ivy.xml      NOTICE.txt  webapps
build.xml    docs                     hadoop-minicluster-1.2.1.jar  lib          README.txt
c++          hadoop-ant-1.2.1.jar     hadoop-test-1.2.1.jar         libexec      sbin
CHANGES.txt  hadoop-client-1.2.1.jar  hadoop-tools-1.2.1.jar        LICENSE.txt  share
conf         hadoop-core-1.2.1.jar    ivy                           logs         src

[root@hadoop01 hadoop]# hadoop fs -copyFromLocal README.txt /input

[root@hadoop01 hadoop]# hadoop fs -ls
ls: Cannot access .: No such file or directory.

[root@hadoop01 hadoop]# hadoop fs -ls /input
Found 1 items
-rw-r--r--   1 root supergroup          0 2015-07-03 23:27 /input/README.txt

[root@hadoop01 hadoop]# hadoop fs -rm /input/README.txt
Deleted hdfs://hadoop01:9000/input/README.txt

[root@hadoop01 hadoop]# hadoop fs -ls /input
Found 1 items
-rw-r--r--   1 root supergroup       1366 2015-07-05 01:06 /input/README.txt

[root@hadoop01 hadoop]# hadoop jar hadoop-examples-1.2.1.jar wordcount /input/README.txt /output
15/07/05 01:16:56 INFO input.FileInputFormat: Total input paths to process : 1
15/07/05 01:16:56 INFO util.NativeCodeLoader: Loaded the native-hadoop library
15/07/05 01:16:56 WARN snappy.LoadSnappy: Snappy native library not loaded
15/07/05 01:16:56 INFO mapred.JobClient: Running job: job_201507050104_0001
15/07/05 01:16:57 INFO mapred.JobClient:  map 0% reduce 0%
15/07/05 01:17:01 INFO mapred.JobClient:  map 100% reduce 0%
15/07/05 01:17:09 INFO mapred.JobClient:  map 100% reduce 33%
15/07/05 01:17:10 INFO mapred.JobClient:  map 100% reduce 100%
15/07/05 01:17:10 INFO mapred.JobClient: Job complete: job_201507050104_0001
15/07/05 01:17:10 INFO mapred.JobClient: Counters: 29
15/07/05 01:17:10 INFO mapred.JobClient:   Job Counters
15/07/05 01:17:10 INFO mapred.JobClient:     Launched reduce tasks=1
15/07/05 01:17:10 INFO mapred.JobClient:     SLOTS_MILLIS_MAPS=3868
15/07/05 01:17:10 INFO mapred.JobClient:     Total time spent by all reduces waiting after reserving slots (ms)=0
15/07/05 01:17:10 INFO mapred.JobClient:     Total time spent by all maps waiting after reserving slots (ms)=0
15/07/05 01:17:10 INFO mapred.JobClient:     Launched map tasks=1
15/07/05 01:17:10 INFO mapred.JobClient:     Data-local map tasks=1
15/07/05 01:17:10 INFO mapred.JobClient:     SLOTS_MILLIS_REDUCES=8429
15/07/05 01:17:10 INFO mapred.JobClient:   File Output Format Counters
15/07/05 01:17:10 INFO mapred.JobClient:     Bytes Written=1306
15/07/05 01:17:10 INFO mapred.JobClient:   FileSystemCounters
15/07/05 01:17:10 INFO mapred.JobClient:     FILE_BYTES_READ=1836
15/07/05 01:17:10 INFO mapred.JobClient:     HDFS_BYTES_READ=1468
15/07/05 01:17:10 INFO mapred.JobClient:     FILE_BYTES_WRITTEN=110060
15/07/05 01:17:10 INFO mapred.JobClient:     HDFS_BYTES_WRITTEN=1306
15/07/05 01:17:10 INFO mapred.JobClient:   File Input Format Counters
15/07/05 01:17:10 INFO mapred.JobClient:     Bytes Read=1366
15/07/05 01:17:10 INFO mapred.JobClient:   Map-Reduce Framework
15/07/05 01:17:10 INFO mapred.JobClient:     Map output materialized bytes=1836
15/07/05 01:17:10 INFO mapred.JobClient:     Map input records=31
15/07/05 01:17:10 INFO mapred.JobClient:     Reduce shuffle bytes=1836
15/07/05 01:17:10 INFO mapred.JobClient:     Spilled Records=262
15/07/05 01:17:10 INFO mapred.JobClient:     Map output bytes=2055
15/07/05 01:17:10 INFO mapred.JobClient:     Total committed heap usage (bytes)=218103808
15/07/05 01:17:10 INFO mapred.JobClient:     CPU time spent (ms)=1140
15/07/05 01:17:10 INFO mapred.JobClient:     Combine input records=179
15/07/05 01:17:10 INFO mapred.JobClient:     SPLIT_RAW_BYTES=102
15/07/05 01:17:10 INFO mapred.JobClient:     Reduce input records=131
15/07/05 01:17:10 INFO mapred.JobClient:     Reduce input groups=131
15/07/05 01:17:10 INFO mapred.JobClient:     Combine output records=131
15/07/05 01:17:10 INFO mapred.JobClient:     Physical memory (bytes) snapshot=280117248
15/07/05 01:17:10 INFO mapred.JobClient:     Reduce output records=131
15/07/05 01:17:10 INFO mapred.JobClient:     Virtual memory (bytes) snapshot=1782865920
15/07/05 01:17:10 INFO mapred.JobClient:     Map output records=179

[root@hadoop01 hadoop]# hadoop fs -ls /output
Found 3 items
-rw-r--r--   1 root supergroup          0 2015-07-05 01:17 /output/_SUCCESS
drwxr-xr-x   - root supergroup          0 2015-07-05 01:16 /output/_logs
-rw-r--r--   1 root supergroup       1306 2015-07-05 01:17 /output/part-r-00000

[root@hadoop01 hadoop]# hadoop fs -cat /output/part-r-00000
(BIS),  1
(ECCN)  1
(TSU)   1
(see    1
5D002.C.1,      1
740.13) 1
    1
Administration  1
Apache  1
BEFORE  1
BIS     1
Bureau  1
Commerce,       1
Commodity       1
Control 1
Core    1
Department      1
ENC     1
Exception       1
Export  2
For     1
Foundation      1
Government      1
Hadoop  1
Hadoop, 1
Industry        1
Jetty   1
License 1
Number  1
Regulations,    1
SSL     1
Section 1
Security        1
See     1
Software        2
Technology      1
The     4
This    1
U.S.    1
Unrestricted    1
about   1
algorithms.     1
and     6
and/or  1
another 1
any     1
as      1
asymmetric      1
at:     2
both    1
by      1
check   1
classified      1
code    1
code.   1
concerning      1
country 1
country's       1
country,        1
cryptographic   3
currently       1
details 1
distribution    2
eligible        1
encryption      3
exception       1
export  1
following       1
for     3
form    1
from    1
functions       1
has     1
have    1
http://hadoop.apache.org/core/  1
http://wiki.apache.org/hadoop/  1
if      1
import, 2
in      1
included        1
includes        2
information     2
information.    1
is      1
it      1
latest  1
laws,   1
libraries       1
makes   1
manner  1
may     1
more    2
mortbay.org.    1
object  1
of      5
on      2
or      2
our     2
performing      1
permitted.      1
please  2
policies        1
possession,     2
project 1
provides        1
re-export       2
regulations     1
reside  1
restrictions    1
security        1
see     1
software        2
software,       2
software.       2
software:       1
source  1
the     8
this    3
to      2
under   1
use,    2
uses    1
using   2
visit   1
website 1
which   2
wiki,   1
with    1
written 1
you     1
your    1



Modify Delete Post Reply Backward Forward List
Powered by Kang Jul Ki