From a550cf151e8fc6626e788fb2bdc72cf03565ff8b Mon Sep 17 00:00:00 2001
From: John Tsiombikas <nuclear@mutantstargoat.com>
Date: Mon, 5 Dec 2016 21:23:24 +0200
Subject: [PATCH] data pipeline

---
 Makefile           |    4 ++++
 tools/prepare_data |   55 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100755 tools/prepare_data

diff --git a/Makefile b/Makefile
index 85c4046..71a41d1 100644
--- a/Makefile
+++ b/Makefile
@@ -43,3 +43,7 @@ clean:
 .PHONY: cleandep
 cleandep:
 	rm -f $(dep)
+
+.PHONY: data
+data:
+	tools/prepare_data
diff --git a/tools/prepare_data b/tools/prepare_data
new file mode 100755
index 0000000..76ced9f
--- /dev/null
+++ b/tools/prepare_data
@@ -0,0 +1,55 @@
+#!/bin/sh
+
+indir=datasrc
+descfile=$indir/datadesc
+outdir=data
+
+process()
+{
+    src_path=$1
+    dst_path=$2
+    op=$3
+
+    cmd=$(echo $op | sed 's/<SRC>/$src_path/g;s/<DST>/$dst_path/g')
+    export src_path
+    export dst_path
+    if ! eval $cmd; then
+		fname=$(basename $src_path)
+		echo "Failed to process data file $fname" >&2
+		echo "Command line was: \"$cmd\""
+		exit 1
+	fi
+}
+
+copy()
+{
+	src_path=$1
+	dst_path=$2
+
+	if ! cp $src_path $dst_path; then
+		fname=$(basename $src_path)
+		echo "Failed to copy data file $fname" >&2
+		exit 1
+	fi
+}
+
+mkdir -p $outdir
+
+while read line; do
+    line=$(echo $line | sed 's/#.*$//')
+    if [ -n "$line" ]; then
+        path=$(echo $line | awk -F : '{ print $1; }')
+        fname=$(basename $path)
+        op=$(echo $line | awk -F : '{ print $2; }' | xargs)
+
+        if [ "$op" = nop ]; then
+            echo copying $fname
+            mkdir -p $outdir/$(dirname $path)
+            copy $indir/$path $outdir/$path
+        else
+            echo processing $fname
+            mkdir -p $outdir/$(dirname $path)
+            process $indir/$path $outdir/$path "$op"
+        fi
+    fi
+done <$descfile
-- 
1.7.10.4