bookdl.pl - cleaned up filter function. added code to download thread. added comments

perl-script/.gitignore

@@ -1 +1,2 @@
-*.json
+*.json
+*.json.[0-9]*

perl-script/bookdl.pl

@@ -7,23 +7,50 @@ use strict;
 
 use JSON;
 
+my $DOMAIN;
 my $THREAD_NO;
+my $thread_no;
 my $URL_PREFIX;
 
-my $FN;
+my $fn;
 my $OUT_DIR;
 
+my $USAGE;
+my $ACMD;
+
 my %jh;
 
 my @a1;
 
-$URL_PREFIX = "https://lainchan.org/lit/src/";
-$THREAD_NO = 4953;
-$FN = "$THREAD_NO.json";
+$DOMAIN = "lainchan.org";
+$URL_PREFIX = "https://$DOMAIN/lit/res/";
 $OUT_DIR = "./dl/";
 
+$USAGE = "Usage: bookdl.pl [http://someurl/]thread_id";
+
+
+# die if no arguments
+die "$USAGE" unless scalar @ARGV > 0;
+$ACMD = $ARGV[0];
+
+($thread_no) = $ACMD =~ /.*\/([0-9]+).{0,5}/;
+
+# die if didnt provide thread number
+die "$USAGE" unless $thread_no;
+
+
+# Download the JSON from lainchan with list
+# of book filenames
+do {
+    my $url = $URL_PREFIX.$thread_no.".json";
+    printf("%s\n", $url);
+    `wget $url -O $thread_no.json`;
+};
+
+$fn = "$thread_no.json";
+
 # Read JSON with list of files
-open FILE, "<", $FN or die "could not open file";
+open FILE, "<", $fn or die "could not open file";
 do{
     my $json_str;
 
@@ -36,7 +63,12 @@ do{
 };
 close FILE;
 
-# anonymous function that returns a list
+# Now we need to parse the JSON we just read
+# into an array of tuples
+
+
+
+# this anonymous function returns a list
 # of tuples of the below form:
 #     (file_name, file_url)
 @a1 = sub{
@@ -47,24 +79,17 @@ close FILE;
 
     # filters for file types we
     # dont want to downloads
-    sub f1 {
-	return $_[0]->{"ext"} && !($_[0]->{"ext"} =~ /jpe?g/);
-    }
-    sub f2 {
-	return !($_[0]->{"ext"} =~ /png/);
-    }
-    sub f3 {
-	return !($_[0]->{"ext"} =~ /gif/);
-    }
-    sub f4 {
-	return !($_[0]->{"ext"} =~ /webm/);
-    }
     sub f0 {
-	return f1($_[0]) && f2($_[0]) && f3($_[0]) && f4($_[0])	
+	return $_[0]->{"ext"} &&
+	    !($_[0]->{"ext"} =~ /jpe?g/) &&
+	    !($_[0]->{"ext"} =~ /png/)   &&
+	    !($_[0]->{"ext"} =~ /gif/)   &&
+	    !($_[0]->{"ext"} =~ /webm/)
     }
 
     # create an array of files
     # that meet our file ext requirement
+    # from the posts attribute
     @a0 = grep {f0($_)} @{$jh{"posts"}};
 
     # do the same filter on the
@@ -88,5 +113,5 @@ close FILE;
 
 # Print a list of wget commands from our tuples
 for my $i1 (@a1){
-    printf("wget -N %s -O '%s%s'\n", scalar $i1->[1], $OUT_DIR, scalar $i1->[0]);
+    printf("wget -nv -nc %s -O '%s%s'\n", scalar $i1->[1], $OUT_DIR, scalar $i1->[0]);
 }

perl-script/user_cleanup.pl

@@ -0,0 +1,32 @@
+#!/usr/bin/perl
+
+use warnings;
+use strict;
+
+my $fn;
+my $cmd_out;
+
+my $dn;
+my @hd;
+
+$dn = "/home/";
+    
+opendir MYDIR, $dn or die "could not open $dn";
+@hd = readdir MYDIR;
+closedir MYDIR;
+
+@hd = grep {!($_ =~ /^\..*$/)} @hd;
+
+$fn = "/tilde/lyadmin/user_list.txt";
+
+open FILE, "<", $fn or die "could not open $fn";
+
+while(<FILE>){
+    chomp;
+
+    unless(exists( {map { $_ => 1} @hd}->{$_})){
+	printf("Deleting user %s\n", $_);
+	`userdel $_`;
+    }
+}
+close FILE