进击大数据系列（十四）Hadoop 数据分析引擎 Apache Pig

例:{（Raja，30），（Mohammad，45）}

例:{Raja，30， {9848022338，raja@gmail.com，} }

例:[name＃Raja，age＃30]

tar -zxvf pig-0.17.0.tar.gz -C ~/training/

PIG_HOME=/root/training/pig-0.17.0
export PIG_HOME

# 本地模式不需要，但是集群模式需要的变量
PIG_CLASSPATH=$HADOOP_HOME/etc/hadoop
export PIG_CLASSPATH

$ pig –version 
 
Apache Pig version 0.17.0 (r1682971)  
compiled Jun 01 2023, 11:44:35

pig -x local

pig

grunt> sh shell command parameters

grunt> sh ls
   
pig 
pig_1444799121955.log 
pig.cmd 
pig.py

grunt> sh File System command parameters

grunt> fs –ls
  
Found 3 items
drwxrwxrwx   - Hadoop supergroup          0 2015-09-08 14:13 Hbase
drwxr-xr-x   - Hadoop supergroup          0 2015-09-09 14:52 seqgen_data
drwxr-xr-x   - Hadoop supergroup          0 2015-09-08 11:30 twitter_data

grunt> clear

grunt> help

Commands: <pig latin statement>; - See the PigLatin manual for details:
http://hadoop.apache.org/pig
  
File system commands:fs <fs arguments> - Equivalent to Hadoop dfs  command:
http://hadoop.apache.org/common/docs/current/hdfs_shell.html
  
Diagnostic Commands:describe <alias>[::<alias] - Show the schema for the alias.
Inner aliases can be described as A::B.
    explain [-script <pigscript>] [-out <path>] [-brief] [-dot|-xml] 
       [-param <param_name>=<pCram_value>]
       [-param_file <file_name>] [<alias>] - 
       Show the execution plan to compute the alias or for entire script.
       -script - Explain the entire script.
       -out - Store the output into directory rather than print to stdout.
       -brief - Don't expand nested plans (presenting a smaller graph for overview).
       -dot - Generate the output in .dot format. Default is text format.
       -xml - Generate the output in .xml format. Default is text format.
       -param <param_name - See parameter substitution for details.
       -param_file <file_name> - See parameter substitution for details.
       alias - Alias to explain.
       dump <alias> - Compute the alias and writes the results to stdout.

Utility Commands: exec [-param <param_name>=param_value] [-param_file <file_name>] <script> -
       Execute the script with access to grunt environment including aliases.
       -param <param_name - See parameter substitution for details.
       -param_file <file_name> - See parameter substitution for details.
       script - Script to be executed.
    run [-param <param_name>=param_value] [-param_file <file_name>] <script> -
       Execute the script with access to grunt environment.
   -param <param_name - See parameter substitution for details.         
       -param_file <file_name> - See parameter substitution for details.
       script - Script to be executed.
    sh  <shell command> - Invoke a shell command.
    kill <job_id> - Kill the hadoop job specified by the hadoop job id.
    set <key> <value> - Provide execution parameters to Pig. Keys and values are case sensitive.
       The following keys are supported:
       default_parallel - Script-level reduce parallelism. Basic input size heuristics used 
       by default.
       debug - Set debug on or off. Default is off.
       job.name - Single-quoted name for jobs. Default is PigLatin:<script name>     
       job.priority - Priority for jobs. Values: very_low, low, normal, high, very_high.
       Default is normal stream.skippath - String that contains the path.
       This is used by streaming any hadoop property.
    help - Display this message.
    history [-n] - Display the list statements in cache.
       -n Hide line numbers.
    quit - Quit the grunt shell. 

grunt> customers = LOAD 'hdfs://localhost:9000/pig_data/customers.txt' USING PigStorage(',');
 
grunt> orders = LOAD 'hdfs://localhost:9000/pig_data/orders.txt' USING PigStorage(',');
 
grunt> student = LOAD 'hdfs://localhost:9000/pig_data/student.txt' USING PigStorage(',');

grunt> history

customers = LOAD 'hdfs://localhost:9000/pig_data/customers.txt' USING PigStorage(','); 
  
orders = LOAD 'hdfs://localhost:9000/pig_data/orders.txt' USING PigStorage(',');
   
student = LOAD 'hdfs://localhost:9000/pig_data/student.txt' USING PigStorage(',');

grunt> quit

grunt> exec [–param param_name = param_value] [–param_file file_name] [script]

Student.txt

001,Rajiv,Hyderabad
002,siddarth,Kolkata
003,Rajesh,Delhi

Sample_script.pig

student = LOAD 'hdfs://localhost:9000/pig_data/student.txt' USING PigStorage(',') 
   as (id:int,name:chararray,city:chararray);
  
Dump student;

grunt> exec /sample_script.pig

(1,Rajiv,Hyderabad)
(2,siddarth,Kolkata)
(3,Rajesh,Delhi)

grunt> kill JobId

grunt> kill Id_0055

grunt> run [–param param_name = param_value] [–param_file file_name] script

Student.txt

001,Rajiv,Hyderabad
002,siddarth,Kolkata
003,Rajesh,Delhi

Sample_script.pig

student = LOAD 'hdfs://localhost:9000/pig_data/student.txt' USING
   PigStorage(',') as (id:int,name:chararray,city:chararray);

grunt> run /sample_script.pig

grunt> Dump;

(1,Rajiv,Hyderabad)
(2,siddarth,Kolkata)
(3,Rajesh,Delhi)

create table tmp_table(name varchar(32), age int, is_child boolean);
create table tmp_table2(age int, user varchar(50), is_child boolean);
load data local infile'/tmp/data_file1' into table tmp_table;
load data local infile'/tmp/data_file2' into table tmp_table2;

tmp_table  = load'/tmp/data_file1' using PigStorage('\t') as(name:chararray, age:int, is_child:int);
tmp_table2 = load'/tmp/data_file2' using PigStorage('\t') as(age:int, user:chararray, is_child:int);

select * from tmp_table limit 50;

tmp_table_limit = limit tmp_table 50;
dump tmp_table_limit;

select name from tmp_table;

tmp_table_name = foreach tmp_table generate name;
dump tmp_table_name;

select name as username, age as userage from tmp_table;

tmp_table_column_alias = foreach tmp_table generate name as username, age as userage;
dump tmp_table_column_alias;

select * from tmp_table order by age;

tmp_table_order = order tmp_table by age asc;
dump tmp_table_order;

select * from tmp_table where age > 18;

tmp_table_where = filter tmp_table by age > 18;
dump tmp_table_where;

select * from tmp_table A join tmp_table2 B on A.age = B.age;

tmp_table_inner_join = join tmp_table by age, tmp_table2 by age;
dump tmp_table_inner_join;

select * from tmp_table A left join tmp_table2 B on A.age = B.age;

tmp_table_left_join = join tmp_table by age left outer, tmp_table2 by age;
dump tmp_table_left_join;

select * from tmp_table A right join tmp_table2 B on A.age = B.age;

tmp_table_right_join = join tmp_table by age right outer,tmp_table2 by age;
dump tmp_table_right_join;

select * from tmp_table A join tmp_table2 B on A.age = B.age Union select * from tmp_table A left join tmp_table2 B on A.age = B.age;

tmp_table_full_join = join tmp_table by age full outer,tmp_table2 by age;
dump tmp_table_full_join;

select * from tmp_table,tmp_table2;

tmp_table_cross = cross tmp_table,tmp_table2;
dump tmp_table_cross ;

select * from tmp_table group by is_child;

tmp_table_group = group tmp_table by is_child;
dump tmp_table_group;

select is_male, count(*) from tmp_table group by is_child;

tmp_table_group_count = group tmp_table by is_child;
tmp_table_group_count = foreach tmp_table_group_count generate group,count($1);
dump tmp_table_group_count;

select distinct is_child from tmp_table;

tmp_table_distinct = foreach tmp_table generate is_child;
tmp_table_distinct = distinct tmp_table_distinct;
dump tmp_table_distinct;