Shiz
/
kiekus
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix bugs, update RevSpace config

This commit is contained in:
Shiz 2022-08-14 16:50:07 +02:00
parent 18efa3a277
commit 0b08ad51f2
6 changed files with 39 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
conf/kiekus-revspace-tz71.conf
View File

@ -1,7 +1,8 @@
devtype=blk
devargs=sd*
devargs=sd*[0-9]
devsuffix=/DCIM/105_PANA
destproto=lftp
dest=sftp://kiekjes@mediaserver/photos
dest=sftp://wipcamera:unused@mediaserver//photos/wip
rpi_hub_port=2
camera_sleep_delay=120

9
kieklist-file
View File

@ -3,7 +3,12 @@ set -e
for dir; do
# disregard the 'total' header and symlink references
ls -apl "$dir" | sed -ne '2,$p' | sed -e 's/ -> .*$//g' | while read -r mode nlinks user group size month day yearortime filename; do
rp-make "$filename" "size=$size" "mode=$mode" "user=$user" "group=$group" "timestamp=$(date -r "$dir/$filename" "+%Y-%m-%dT%H:%I:%S")"
ls -apl "$dir" | grep -v '^total' | sed -e 's/ -> .*$//g' | while read -r mode nlinks user group size month day yearortime filename; do
statfile="$dir/$filename"
case "$filename" in
*/);;
*/*) statfile="$filename"; filename="$(basename "$filename")";;
esac
rp-make "$filename" "size=$size" "mode=$mode" "user=$user" "group=$group" "timestamp=$(date -r "$statfile" "+%Y-%m-%dT%H:%I:%S")"
done
done

2
kieklist-lftp
View File

@ -2,7 +2,7 @@
set -e
for target; do
lftp -c "connect $target; cls -l --time-style='%Y-%m-%dT%H:%I:%S'" | while read -r mode nlinks user group size datetime filename; do
lftp -c "connect $target; cls -al --time-style='%Y-%m-%dT%H:%I:%S'" | while read -r mode nlinks user group size datetime filename; do
rp-make "$filename" "size=$size" "mode=$mode" "user=$user" "group=$group" "timestamp=$datetime"
done
done

2
kiekpull-file
View File

@ -2,5 +2,5 @@
set -e
basedir="$1"; shift
[ -d "$basedir" ] || exit 1
[ "$#" -gt 1 ] || set -- .
[ "$#" -ge 1 ] || set -- .
(cd "$basedir" && tar -cz -- "$@")

1
kiekpush-lftp
View File

@ -10,6 +10,7 @@ directory="${1#$server/}"
(cd "$tempdir"; tar -xz)
while read -r filename; do
[ -n "$filename" ] || continue
lftp -c "connect $server && mkdir -pf $directory && cd $directory && lcd $tempdir && put $filename"
printf "%s\n" "$filename"
done <<EOF

37
kiekus
View File

@ -20,6 +20,7 @@ eval "$(kiekconf \
${cfgfile:+-c "$cfgfile"} \
devtype=blk \
devargs= \
devsuffix= \
destproto=lftp \
dest="$dest" \
statedir=/tmp/kiekstate.$$ \
@ -31,34 +32,48 @@ eval "$(kiekconf \
iso8601_to_touch() {
printf "%s\n" | sed -E -e 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2}).*/\1\2\3\4\5.\6/g'
printf "%s\n" "$1" | sed -E -e 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2}).*/\1\2\3\4\5.\6/g'
}
mkdir -p "$statedir"
eval "kiekmon-$devtype $devargs" | rp-get . protocol rm_command | while read -r device && read -r srcproto && read -r rmcmd; do (
trap 'set +e; code=$?; [ -z "$posthook" ] || eval "$posthook"; eval "$rmcmd"; exit $code' EXIT INT TERM
eval "kiekmon-$devtype $devargs" | while read -r line; do (
read -r rmcmd <<EOF
$(printf "%s\n" "$line" | rp-get rm_command)
EOF
trap 'exit $?' INT TERM ERR
trap 'code=$?; set +e; [ -z "$posthook" ] || eval "$posthook"; eval "$rmcmd"; exit $code' EXIT
{ read -r srcproto; read -r device; } <<EOF
$(printf "%s\n" "$line" | rp-get protocol)
$(printf "%s\n" "$line" | rp-get .)
EOF
[ -z "$prehook" ] || eval "$prehook"
# determine destination directory and last upload timestamp
destdir="$dest/$(date +"%Y%m%d")"
kiekpull-$destproto "$dest" "$logfilename" | kiekpush-file "$statedir" || true
if [ -f "$statedir/$logfilename" ]; then
lastts=$(date -r "$statedir/$logfilename" +"%Y-%m-%d%H:%I:%S")
lastts=$(date -r "$statedir/$logfilename" +"%Y-%m-%dT%H:%I:%S")
else
# best-effort
lastts="$(kieklist-$destproto "$dest" | rp-filter '.= | endswith("/")' | rp-sort -r . | rp-get timestamp)"
lastts="$(kieklist-$destproto "$dest" | rp-filter '.= | endswith("/")' | rp-filter '.= != "../"' | rp-sort -r . | rp-get timestamp | head -n 1)"
fi
# get source file listing and filter it
filenames="$(kieklist-$srcproto "$device" | rp-filter "timestamp= > $lastts" | rp-get .)"
files="$(kieklist-$srcproto "$device$devsuffix" | rp-filter "timestamp= > \"$lastts\"")"
filenames="$(printf "%s\n" "$files" | rp-get . | grep -v -e '^./$' -e '^../$')"
[ -f "$statedir/$logfilename" ] && filenames=$(printf "%s\n" "$filenames" | grep -vF -f "$statedir/$logfilename")
# pull from source and push into destination
kiekpull-$srcproto "$device" $filenames | kiekpush-$destproto "$destdir" | while read -r line; do
printf "%s\n" "$line" >> "$statedir/$logfilename"
done
if [ -n "$filenames" ]; then
lastts="$(printf "%s\n" "$files" | rp-get timestamp | sort -r | head -n 1)"
kiekpull-$srcproto "$device$devsuffix" $filenames | kiekpush-$destproto "$destdir" | while read -r line; do
printf "%s\n" "$line" >> "$statedir/$logfilename"
done
fi
# update logfile timestamp, upload logfile
touch -t $(iso8601_to_touch "$lastts") "$logfilename"
kiekpull-file "$statedir" "$logfilename" | kiekpush-$destproto "$dest"
if [ -n "$filenames" ] || ! [ -f "$statedir/$logfilename" ]; then
touch -t $(iso8601_to_touch "$lastts") "$statedir/$logfilename"
kiekpull-file "$statedir" "$logfilename" | kiekpush-$destproto "$dest"
fi
) || true; done