data pipeline

diff --git a/Makefile b/Makefile
index 85c4046..71a41d1 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -43,3 +43,7 @@ clean:
 .PHONY: cleandep
 cleandep:
        rm -f $(dep)
+
+.PHONY: data
+data:
+       tools/prepare_data

diff --git a/tools/prepare_data b/tools/prepare_data
new file mode 100755 (executable)
index 0000000..76ced9f
--- /dev/null
+++ b/tools/prepare_data
@@ -0,0 +1,55 @@
+#!/bin/sh
+
+indir=datasrc
+descfile=$indir/datadesc
+outdir=data
+
+process()
+{
+    src_path=$1
+    dst_path=$2
+    op=$3
+
+    cmd=$(echo $op | sed 's/<SRC>/$src_path/g;s/<DST>/$dst_path/g')
+    export src_path
+    export dst_path
+    if ! eval $cmd; then
+               fname=$(basename $src_path)
+               echo "Failed to process data file $fname" >&2
+               echo "Command line was: \"$cmd\""
+               exit 1
+       fi
+}
+
+copy()
+{
+       src_path=$1
+       dst_path=$2
+
+       if ! cp $src_path $dst_path; then
+               fname=$(basename $src_path)
+               echo "Failed to copy data file $fname" >&2
+               exit 1
+       fi
+}
+
+mkdir -p $outdir
+
+while read line; do
+    line=$(echo $line | sed 's/#.*$//')
+    if [ -n "$line" ]; then
+        path=$(echo $line | awk -F : '{ print $1; }')
+        fname=$(basename $path)
+        op=$(echo $line | awk -F : '{ print $2; }' | xargs)
+
+        if [ "$op" = nop ]; then
+            echo copying $fname
+            mkdir -p $outdir/$(dirname $path)
+            copy $indir/$path $outdir/$path
+        else
+            echo processing $fname
+            mkdir -p $outdir/$(dirname $path)
+            process $indir/$path $outdir/$path "$op"
+        fi
+    fi
+done <$descfile