Generic HDFS path in external Hive tables

2-import/README

-1. 1. 2015 ... 1.1.2016
-1.1.2016 ... 1.1.2017
-
-1420070400 ... 1451606400
-1451606400 ... 1483228800
-
-SELECT id, name, user, submit, start, finish, memory_seconds, cpu_seconds, nmap, nreduce FROM jobs WHERE start >= 1420070400L*1000 AND finish < 1451606400L*1000;

2-import/hive-import.sh

@@ -26,8 +26,8 @@ for action in ${ACTIONS}; do
       hdfs dfs -rm -r ${HDFS_PATH} || :
       hdfs dfs -mkdir -p ${HDFS_PATH}/jobs/
       for t in ${TABLES}; do
-        hdfs dfs -mkdir /user/valtri/statistics/jobs/${t} || : 2>/dev/null
-        hdfs dfs -put ${LOCAL_PATH}/${t}.csv /user/valtri/statistics/jobs/${t}/
+        hdfs dfs -mkdir ${HDFS_PATH}/jobs/${t} || : 2>/dev/null
+        hdfs dfs -put ${LOCAL_PATH}/${t}.csv ${HDFS_PATH}/jobs/${t}/
       done
       ;;
 
@@ -36,8 +36,9 @@ for action in ${ACTIONS}; do
       ;;
 
     init)
+      sed -e "s,@HDFS_PATH@,$HDFS_PATH,g" "${BASE_PATH}/hive.sql.in" > hive.sql
       beeline -u $JDBC_URL -e "CREATE DATABASE $DBNAME"
-      beeline -u $JDBC_URL -f "${BASE_PATH}/hive.sql"
+      beeline -u $JDBC_URL -f hive.sql
       ;;
 
   esac

2-import/hive.sql → 2-import/hive.sql.in

@@ -16,7 +16,7 @@ CREATE EXTERNAL TABLE jobs (
     changed TIMESTAMP
 )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
-STORED AS TEXTFILE LOCATION '/user/valtri/statistics/jobs/jobs'
+STORED AS TEXTFILE LOCATION '@HDFS_PATH@/jobs/jobs'
 TBLPROPERTIES("skip.header.line.count"="1");
 
 
@@ -30,7 +30,7 @@ CREATE EXTERNAL TABLE subjobs (
     finish BIGINT
 )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
-STORED AS TEXTFILE LOCATION '/user/valtri/statistics/jobs/subjobs'
+STORED AS TEXTFILE LOCATION '@HDFS_PATH@/jobs/subjobs'
 TBLPROPERTIES("skip.header.line.count"="1");
 
 
@@ -42,7 +42,7 @@ CREATE EXTERNAL TABLE jobnodes (
     nreduce INT
 )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
-STORED AS TEXTFILE LOCATION '/user/valtri/statistics/jobs/jobnodes'
+STORED AS TEXTFILE LOCATION '@HDFS_PATH@/jobs/jobnodes'
 TBLPROPERTIES("skip.header.line.count"="1");
 
 
@@ -55,7 +55,7 @@ CREATE EXTERNAL TABLE jobcounters (
     total BIGINT
 )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
-STORED AS TEXTFILE LOCATION '/user/valtri/statistics/jobs/jobcounters'
+STORED AS TEXTFILE LOCATION '@HDFS_PATH@/jobs/jobcounters'
 TBLPROPERTIES("skip.header.line.count"="1");
 
 
@@ -65,7 +65,7 @@ CREATE EXTERNAL TABLE counters (
     name CHAR(128)
 )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
-STORED AS TEXTFILE LOCATION '/user/valtri/statistics/jobs/counters'
+STORED AS TEXTFILE LOCATION '@HDFS_PATH@/jobs/counters'
 TBLPROPERTIES("skip.header.line.count"="1");
 
 
@@ -74,7 +74,7 @@ CREATE EXTERNAL TABLE nodes (
     host VARCHAR(256)
 )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
-STORED AS TEXTFILE LOCATION '/user/valtri/statistics/jobs/nodes'
+STORED AS TEXTFILE LOCATION '@HDFS_PATH@/jobs/nodes'
 TBLPROPERTIES("skip.header.line.count"="1");
 
 
@@ -83,5 +83,5 @@ CREATE EXTERNAL TABLE intervals (
     finish INT
 )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
-STORED AS TEXTFILE LOCATION '/user/valtri/statistics/jobs/intervals/'
+STORED AS TEXTFILE LOCATION '@HDFS_PATH@/jobs/intervals/'
 TBLPROPERTIES("skip.header.line.count"="1");