Offline mode for shimbun retrieval

[David <deng@xxxxxxxxxxxxxxx>]

I'm in the process of completely converting from nnrss to nnshimbun in
Gnus. One feature that I like in nnrss is the possibility to use it in
"local mode", where an external script downloads the feeds. This script
can be called asynchronously from Emacs or regularly through cron, which
speeds up checking the groups considerably.

I've implemented this for emacs-w3m/shimbun; the patch is attached to
this mail. It also contains a function for generating a download shell
script, which is specific to Gnus/nnshimbun, but I guess it should be
possible to create something similar for the other Emacs mail
readers. For example, a script for downloading the emacswiki RSS looks
like this:

,----
| #!/bin/sh
| # shimbun download script
| 
| W3M=/usr/bin/w3m
| OPTS="-o accept_encoding=identity -dump_both"
| umask 066
| 
| $W3M $OPTS http://www.emacswiki.org/emacs/full.rss > /tmp/465e89d887_shimbun
`----

The filenames used for saving the shimbuns are generated through a md5
of the URL (truncated to the first 10 chars). The default directory is
/tmp, but this can be customized. If a file for a URL is not available,
it is retrieved as usual.

Regards,
David

Index: shimbun.el
===================================================================
RCS file: /storage/cvsroot/emacs-w3m/shimbun/shimbun.el,v
retrieving revision 1.194
diff -u -r1.194 shimbun.el
--- shimbun.el	23 Jul 2008 08:25:51 -0000	1.194
+++ shimbun.el	25 Nov 2008 20:37:50 -0000
@@ -185,6 +185,21 @@
 			 :match (lambda (widget value) (natnump value))
 			 :value 1)))
 
+(defcustom shimbun-use-local nil
+  "Specifies if local files should be used (\"offline\" mode).
+This way, you can use an external script to retrieve the
+necessary HTML/XML files.  For an example, see
+`nnshimbun-generate-download-script'.  If a local file for an URL
+cannot be found, it will silently be retrieved as usual."
+  :group 'shimbun
+  :type 'boolean)
+
+(defcustom shimbun-local-path temporary-file-directory
+  "Directory where local shimbun files are stored.
+Default is the system's temporary directory."
+  :group 'shimbun
+  :type 'directory)
+
 (defun shimbun-servers-list ()
   "Return a list of shimbun servers."
   (let (servers)
@@ -225,14 +240,38 @@
   "Rertrieve URL contents and insert to current buffer.
 Return content-type of URL as string when retrieval succeeded.
 Non-ASCII characters `url' are escaped based on `url-coding-system'."
-  (let (type)
-    (if (and url
-	     (setq type (w3m-retrieve
-			 (w3m-url-transfer-encode-string url url-coding-system)
-			 nil no-cache nil referer)))
+  (let (type charset fname)
+    (if (and url 
+	     shimbun-use-local
+	     shimbun-local-path
+	     (file-regular-p 
+	      (setq fname (concat (file-name-as-directory
+				   (expand-file-name shimbun-local-path))
+				  (substring (md5 url) 0 10)
+				  "_shimbun"))))
+	;; get local file contents
+	(progn
+	  (let ((coding-system-for-read 'no-conversion))
+	    (insert-file-contents fname))
+	  (when (re-search-forward "^$" nil t)
+	    (let ((pos (match-beginning 0)))
+	      (re-search-backward 
+	       "^Content-Type: \\(.*?\\)\\(?:[ ;]+\\|$\\)\\(charset=\\(.*\\)\\)?"
+	       nil t)
+	      (setq type (match-string 1)
+		    charset (match-string 3))
+	      (delete-region (point-min) pos))))
+      ;; retrieve URL
+      (when url
+	(setq type (w3m-retrieve
+		    (w3m-url-transfer-encode-string url url-coding-system)
+		    nil no-cache nil referer))))
+    (if type
 	(progn
 	  (unless no-decode
-	    (w3m-decode-buffer url)
+	    (if charset
+		(w3m-decode-buffer url charset type)
+	      (w3m-decode-buffer url))
 	    (goto-char (point-min)))
 	  type)
       (unless no-decode
Index: nnshimbun.el
===================================================================
RCS file: /storage/cvsroot/emacs-w3m/shimbun/nnshimbun.el,v
retrieving revision 1.62
diff -u -r1.62 nnshimbun.el
--- nnshimbun.el	17 Oct 2007 11:15:58 -0000	1.62
+++ nnshimbun.el	25 Nov 2008 20:37:53 -0000
@@ -993,6 +993,34 @@
 		(gnus-group-make-group grp (list 'nnshimbun server)))))
 	(message "No group is found in nnshimbun+%s:" server)))))
 
+(defun nnshimbun-generate-download-script (&optional async)
+  "Generate download script for all subscribed schimbuns.
+Output will be put in a new buffer.  If called with a prefix,
+puts a '&' after each curl command."
+  (interactive "P")
+  (switch-to-buffer
+   (get-buffer-create "*shimbun download script*"))
+  (erase-buffer)
+  (insert (concat "#!/bin/sh\n# shimbun download script\n\n"
+		  "W3M=" (if w3m-command w3m-command "/usr/bin/w3m")
+		  "\nOPTS=\"-o accept_encoding=identity -dump_both\"\n"
+		  "umask 066\n\n"))
+  (let ((path (file-name-as-directory
+		(expand-file-name shimbun-local-path)))
+	url fname)
+    ;; get all subscribed shimbun groups
+    (dolist (cur gnus-newsrc-alist)
+      (when (and (eq (car-safe (nth 4 cur)) 'nnshimbun)
+		 (<= (nth 1 cur) gnus-level-subscribed))
+	(when (string-match "nnshimbun\\+\\(.+\\):\\(.+\\)" (car cur))
+	  (nnshimbun-possibly-change-group (match-string 2 (car cur))
+					   (match-string 1 (car cur)))
+	  (setq url (shimbun-index-url nnshimbun-shimbun))
+	  (setq fname (concat path (substring (md5 url) 0 10) "_shimbun"))
+	  (insert
+	   (concat "$W3M $OPTS " url " > " fname
+		   (if async " &\n" "\n"))))))))
+
 (provide 'nnshimbun)
 
 ;;; nnshimbun.el ends here