* (BETA) Implemented file list exporting and importing

* Added file format specification (README.fileformat)
* Moved settingsCli in settings.c to parseCli in main.c
* Bugfix: drawError in calc.c was not shown when an error occurred while reading the main dir


git-svn-id: svn://blicky.net/ncdu/trunk@25 ce56bc8d-f834-0410-b703-f827bd498a76

README.fileformat

+                         ncdu file list format v1                    11-08-2007
+
+
+This document describes the file format of the file list export feature of
+ncdu. This document was used as a reference for implementing the export
+feature, and may be useful to anyone who wants to read or write these files.
+
+
+
+definitions
+  byte  = unsigned 8bit integer
+  2uint = unsigned 16bit integer
+  4uint = unsigned 32bit integer
+  8int  = signed 64bit integer
+  8uint = unsigned 64bit integer
+
+  All integers are in network (big-endian) byte order.
+
+
+
+header
+ ssssvnttttttttiiii
+
+  s = "ncdu" (hex: 6e 63 64 75)
+  v = version of the file format, always 1 (hex: 01)
+  n = NULL-terminated string containing the package name and version of the
+      program that created this file. ncdu 1.3 would be "ncdu 1.3\0". Maximum
+      of 64 bytes including the terminating NULL byte.
+  t = 8int, unix timestamp of when the file was created
+  i = 4uint, total number of directory items
+
+
+
+directory item
+ llfssssssssaaaaaaaaddddn
+
+  l = 2uint, level. 0 for parent, 1 for sub, 2 for sub sub, etc. (see below)
+  f = byte, flags: (mostly the same as defined in ncdu.h)
+       01  item is a directory
+       02  item is a file
+       04  error while reading this item
+       08  item was excluded because it was on an other filesystem
+       10  item was exluded using exclude patterns
+  s = 8uint, disk usage of current item (st_blocks * 512)
+  a = 8uint, apparent size of current item (st_size)
+  n = NULL-terminated string, name of the current item, absolute maximum length
+      of 8192 bytes including the terminating NULL-byte.
+
+
+
+global layout
+  An ncdu datafile always starts with the header, this header also specifies
+  the number of directory items. All data after the last item should be
+  ignored.
+
+  After the header follows the list of directory items. The first directory
+  item should be the parent directory. This is the only item to have a full and
+  absolute path as name, all other items should only include the name of the
+  item itself. This is also the only item to have a level of 0.
+
+  The second directory item should level set to 1, and should be an item
+  located in the parent directory. Any higher level than the previous item
+  indicates that the item is located in the previous item, the same level
+  indicates this item is located in the same directory as the previous item,
+  and a lower level indicates that this item is located in de same directory
+  as the parent directory of the previous item.
+
+  The disk usage and apparent size of each directory item is accumulated, it
+  should already include the sizes of the subitems.
+
+  Example showing the use of levels and names:
+
+    [header](items=7)
+    [directory item](level=0, name=/parent/dir)       /parent/dir
+    [directory item](level=1, name=file1)             /parent/dir/file1
+    [directory item](level=1, name=dir1)              /parent/dir/dir1
+    [directory item](level=2, name=sfile1)            /parent/dir/dir1/sfile1
+    [directory item](level=2, name=sfile2)            /parent/dir/dir1/sfile2
+    [directory item](level=1, name=file2)             /parent/dir/file2
+    [directory item](level=1, name=file3)             /parent/dir/file3

src/Makefile.am

 bin_PROGRAMS = ncdu
 
-ncdu_SOURCES = browser.c calc.c main.c settings.c util.c exclude.c help.c delete.c
+ncdu_SOURCES = browser.c calc.c main.c settings.c util.c exclude.c help.c delete.c export.c
 
 noinst_HEADERS = ncdu.h

src/calc.c

@@ -192,6 +192,7 @@ static void drawError(char *dir) {
   mvwprintw(err, 5, 9, "could not open %s", cropdir(dir, 34));
   mvwaddstr(err, 6, 3, "press any key to continue...");
 
+  refresh();
   wrefresh(err);
   delwin(err);
 }
@@ -202,6 +203,10 @@ int updateProgress(char *path) {
   struct timeval tv;
   int ch;
 
+ /* don't do anything if s_export is set (ncurses isn't initialized) */
+  if(s_export)
+    return(1);
+
  /* check for input or screen resizes */
   nodelay(stdscr, 1);
   while((ch = getch()) != ERR) {
@@ -374,12 +379,17 @@ struct dir *showCalc(char *path) {
   lastupdate = 999;
 
  /* init parent dir */
-  if(rpath(path, tmp) == NULL || lstat(tmp, &fs) != 0) {
+  if(rpath(path, tmp) == NULL || lstat(tmp, &fs) != 0 || !S_ISDIR(fs.st_mode)) {
+    if(s_export) {
+      printf("Error: could not open %s\n", path);
+      exit(1);
+    }
     do {
       ncresize();
       if(dat != NULL)
         drawBrowser(0);
       drawError(path);
+      refresh();
     } while (getch() == KEY_RESIZE);
     return(NULL);
   }

src/export.c

+/* ncdu - NCurses Disk Usage 
+  
+  Copyright (c) 2007 Yoran Heling
+
+  Permission is hereby granted, free of charge, to any person obtaining
+  a copy of this software and associated documentation files (the
+  "Software"), to deal in the Software without restriction, including
+  without limitation the rights to use, copy, modify, merge, publish,
+  distribute, sublicense, and/or sell copies of the Software, and to
+  permit persons to whom the Software is furnished to do so, subject to
+  the following conditions:
+  
+  The above copyright notice and this permission notice shall be included
+  in all copies or substantial portions of the Software.
+  
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include "ncdu.h"
+
+
+/* !WARNING! There is currently *NO* error handling at all */
+
+
+/*
+ *   E X P O R T I N G
+*/
+
+
+unsigned int ilevel;
+
+
+#define writeInt(hl, word, bytes) _writeInt(hl, (unsigned char *) &word, bytes, sizeof(word))
+
+/* Write any integer in network byte order.
+ * This function always writes the number of bytes specified by storage to the
+ * file, disregarding the actual size of word. If the actual size is smaller
+ * than the storage, the number will be preceded by null-bytes. If the actual
+ * size is greater than the storage, we assume the value we want to store fits
+ * in the storage size, disregarding any overflow.
+*/
+void _writeInt(FILE *wr, unsigned char *word, int storage, int size) {
+  unsigned char buf[8]; /* should be able to store any integer type */
+  int i;
+
+ /* clear the buffer */
+  memset(buf, 0, 8);
+
+ /* store the integer in the end of the buffer, in network byte order */
+  if(IS_BIG_ENDIAN)
+    memcpy(buf+(8-size), word, size);
+  else
+    for(i=0; i<size; i++)
+      buf[8-size+i] = word[size-1-i];
+
+ /* write only the last bytes of the buffer (as specified by storage) to the file */
+  fwrite(buf+(8-storage), 1, storage, wr);
+}
+
+
+/* recursive */
+long calcItems(struct dir *dr) {
+  int count = 0;
+  do {
+    if(dr->flags & FF_PAR)
+      continue;
+    if(dr->sub)
+      count += calcItems(dr->sub);
+    count++;
+  } while((dr = dr->next) != NULL);
+  return(count);
+}
+
+
+/* also recursive */
+void writeDirs(FILE *wr, struct dir *dr) {
+  unsigned char f;
+  
+  do {
+    if(dr->flags & FF_PAR)
+      continue;
+  
+   /* flags - the slow but portable way */
+    f = 0;
+    if(dr->flags & FF_DIR)   f += EF_DIR;
+    if(dr->flags & FF_FILE)  f += EF_FILE;
+    if(dr->flags & FF_ERR)   f += EF_ERR;
+    if(dr->flags & FF_OTHFS) f += EF_OTHFS;
+    if(dr->flags & FF_EXL)   f += EF_EXL;
+
+    writeInt(wr, ilevel, 2);
+    fwrite(&f, 1, 1, wr);
+    writeInt(wr, dr->size, 8);
+    writeInt(wr, dr->asize, 8);
+    fprintf(wr, "%s%c", dr->name, 0);
+
+    if(dr->sub != NULL) {
+      ilevel++;
+      writeDirs(wr, dr->sub);
+      ilevel--;
+    }
+
+  } while((dr = dr->next) != NULL);
+}
+
+
+void exportFile(char *dest, struct dir *src) {
+  FILE *wr;
+  struct timeval tv;
+  long items;
+
+  wr = fopen(dest, "w");
+
+  /* header */
+  fprintf(wr, "ncdu%c%s%c", 1, PACKAGE_STRING, 0);
+
+  gettimeofday(&tv, NULL);
+  writeInt(wr, tv.tv_sec, 8);
+
+  items = calcItems(src);
+  writeInt(wr, items, 4);
+
+  /* the directory items */
+  ilevel = 0;
+  writeDirs(wr, src);
+
+  fclose(wr);
+}
+
+
+
+
+/*
+ *  I M P O R T I N G
+*/
+
+
+#define readInt(hl, word, bytes) _readInt(hl, (unsigned char *) &word, bytes, sizeof(word))
+
+/* reverse of writeInt */
+void _readInt(FILE *rd, unsigned char *word, int storage, int size) {
+  unsigned char buf[8];
+  int i;
+
+ /* clear the buffer */
+  memset(buf, 0, 8);
+
+ /* read integer to the end of the buffer */
+  fread(buf+(8-storage), 1, storage, rd);
+
+ /* copy buf to word, in host byte order */
+  if(IS_BIG_ENDIAN)
+    memcpy(word, buf+(8-size), size);
+  else
+    for(i=0; i<size; i++)
+      word[i] = buf[7-i];
+}
+
+
+struct dir *importFile(char *filename) {
+  unsigned char buf[8];
+  FILE *rd;
+  int i, item;
+  unsigned long items;
+  unsigned int level;
+  struct dir *prev, *cur, *tmp, *parent;
+
+  rd = fopen(filename, "r");
+
+ /* check filetype */
+  fread(buf, 1, 5, rd);
+  if(buf[0] != 'n' || buf[1] != 'c' || buf[2] != 'd'
+      || buf[3] != 'u' || buf[4] != (char) 1)
+    return(NULL);
+
+ /* package name, version and timestamp are ignored for now */
+  for(i=0; i<=64 && fgetc(rd) != 0; i++) ;
+  fread(buf, 1, 8, rd);
+
+ /* number of items is not ignored */
+  readInt(rd, items, 4);
+
+ /* and now start reading the directory items */
+  level = 0;
+  prev = NULL;
+  for(item=0; item<items; item++) {
+    unsigned int curlev;
+    unsigned char flags, name[8192];
+    int ch;
+    
+    readInt(rd, curlev, 2);
+    flags = fgetc(rd);
+
+    cur = calloc(sizeof(struct dir), 1);
+    if(!prev)
+      parent = cur;
+    else if(curlev > level) {
+     /* create a reference to the parent dir... meh, stupid hack */
+      if(curlev > 1) {
+        tmp = calloc(sizeof(struct dir), 1);
+        tmp->name = malloc(3);
+        strcpy(tmp->name, "..");
+        tmp->flags |= FF_PAR;
+        prev->sub = tmp;
+        tmp->next = cur;
+        tmp->parent = prev;
+      } else
+        prev->sub = cur;
+      cur->parent = prev;
+    } else if(curlev == level) {
+      prev->next = cur;
+      cur->parent = prev->parent;
+    } else {
+      for(i=level; i>curlev; i--)
+        prev = prev->parent;
+      prev->next = cur;
+      cur->parent = prev->parent;
+    }
+
+   /* flags - again, the slow but portable way */
+    if(flags & EF_DIR)   cur->flags |= FF_DIR;
+    if(flags & EF_FILE)  cur->flags |= FF_FILE;
+    if(flags & EF_OTHFS) cur->flags |= FF_OTHFS;
+    if(flags & EF_EXL)   cur->flags |= FF_EXL;
+    if(flags & EF_ERR) {
+      cur->flags |= FF_ERR;
+      for(tmp = cur->parent; tmp != NULL; tmp = tmp->parent)
+        tmp->flags |= FF_SERR;
+    }
+
+   /* sizes */
+    readInt(rd, cur->size, 8);
+    readInt(rd, cur->asize, 8);
+
+   /* name */
+    for(i=0; i<8192; i++) {
+      ch = fgetc(rd);
+      name[i] = (unsigned char) ch;
+      if(ch == 0)
+        break;
+    }
+    cur->name = malloc(i+1);
+    strcpy(cur->name, name);
+
+   /* update 'items' of parent dirs */
+    if(!(cur->flags & FF_EXL))
+      for(tmp = cur->parent; tmp != NULL; tmp = tmp->parent)
+        tmp->items++;
+
+    level = curlev;
+    prev = cur;
+  }
+  
+  return(parent);
+}
+
+
+

src/main.c

@@ -28,14 +28,107 @@
 /* check ncdu.h what these are for */
 struct dir *dat;
 int winrows, wincols;
-char sdir[PATH_MAX];
+char sdir[PATH_MAX], *s_export;
 int sflags, bflags, sdelay, bgraph;
 
+
+/* parse command line */
+void parseCli(int argc, char **argv) {
+  int i, j;
+
+ /* load defaults */
+  memset(sdir, 0, PATH_MAX);
+  getcwd(sdir, PATH_MAX);
+  sflags = 0;
+  sdelay = 100;
+  bflags = BF_SIZE | BF_DESC;
+  s_export = NULL;
+  sdir[0] = '\0';
+
+ /* read from commandline */
+  for(i=1; i<argc; i++) {
+    if(argv[i][0] == '-') {
+     /* flags requiring arguments */
+      if(argv[i][1] == 'X' || !strcmp(argv[i], "--exclude-from") || !strcmp(argv[i], "--exclude")
+          || argv[i][1] == 'e' || argv[i][1] == 'l') {
+        if(i+1 >= argc) {
+          printf("Option %s requires an argument\n", argv[i]);
+          exit(1);
+        }
+        if(argv[i][1] == 'e')
+          s_export = argv[++i];
+        else if(strcmp(argv[i], "--exclude") == 0)
+          addExclude(argv[++i]);
+        else if(addExcludeFile(argv[++i])) {
+          printf("Can't open %s: %s\n", argv[i], strerror(errno));
+          exit(1);
+        }
+        continue;
+      }
+     /* small flags */
+      for(j=1; j < strlen(argv[i]); j++)
+        switch(argv[i][j]) {
+          case 'x': sflags |= SF_SMFS; break;
+          case 'q': sdelay = 2000;     break;
+          case '?':
+          case 'h':
+            printf("ncdu [-ahvx] [dir]\n\n");
+            printf("  -h  This help message\n");
+            printf("  -q  Low screen refresh interval when calculating\n");
+            printf("  -v  Print version\n");
+            printf("  -x  Same filesystem\n");
+            exit(0);
+          case 'v':
+            printf("ncdu %s\n", PACKAGE_VERSION);
+            exit(0);  
+          default:
+            printf("Unknown option: -%c\n", argv[i][j]);
+            exit(1);
+        }
+    } else {
+      strcpy(sdir, argv[i]);
+    }
+  }
+  if(s_export && !sdir[0]) {
+    printf("Can't export file list: no directory specified!\n");
+    exit(1);
+  }
+}
+
+
+/* if path is a file: import file list
+ * if path is a dir:  calculate directory */
+struct dir *loadDir(char *path) {
+  struct stat st;
+  
+  if(stat(path, &st) < 0) {
+    if(s_export) {
+      printf("Error: Can't open %s!", path);
+      exit(1);
+    }
+    return(showCalc(path));
+  }
+
+  if(S_ISREG(st.st_mode))
+    return(importFile(path));
+  else
+    return(showCalc(path));
+}
+
+
 /* main program */
 int main(int argc, char **argv) {
-  int gd;
+  dat = NULL;
+
+  parseCli(argc, argv);
+
+ /* only export file, don't init ncurses */
+  if(s_export) {
+    dat = loadDir(sdir);
+    exportFile(s_export, dat);
+    exit(0);
+  }
 
-  gd = settingsCli(argc, argv);
   initscr();
   cbreak();
   noecho();
@@ -43,11 +136,10 @@ int main(int argc, char **argv) {
   keypad(stdscr, TRUE);
   ncresize();
   
-  if(gd && settingsWin())
+  if(!sdir[0] && settingsWin())
     goto mainend;
 
-  dat = NULL;
-  while((dat = showCalc(sdir)) == NULL)
+  while((dat = loadDir(sdir)) == NULL)
     if(settingsWin())
       goto mainend;
 

src/ncdu.h

@@ -75,6 +75,14 @@
 # define S_ISLNK(x) (x & S_IFLNK)
 #endif
 
+/* There are many ways to test the endianness on a system, however, I couldn't
+ * find a universal way to do this, so I used this small hack that should work
+ * on any system. */
+static unsigned int endian_test = 1;
+#define IS_BIG_ENDIAN (!(*(char *) &endian_test))
+
+
+
 /*
  *    G L O B A L   F L A G S
  */
@@ -104,15 +112,23 @@
 #define BF_SORT   0x20 /* no need to resort, list is already in correct order */
 #define BF_AS     0x40 /* show apparent sizes instead of disk usage */
 
+/* Export Flags */
+#define EF_DIR    0x01
+#define EF_FILE   0x02
+#define EF_ERR    0x04
+#define EF_OTHFS  0x08
+#define EF_EXL    0x10
+
+
 
 /*
  *    S T R U C T U R E S
  */
 struct dir {
   struct dir *parent, *next, *sub;
-  char *name;
+  unsigned char *name;
   off_t size, asize;
-  unsigned int items;
+  unsigned long items;
   unsigned char flags;
 }; 
 
@@ -127,7 +143,7 @@ extern struct dir *dat;
 /* updated when window is resized */
 extern int winrows, wincols;
 /* global settings */
-extern char sdir[PATH_MAX];
+extern char sdir[PATH_MAX], *s_export;
 extern int sflags, bflags, sdelay, bgraph;
 
 
@@ -142,7 +158,6 @@ extern void ncresize(void);
 extern struct dir * freedir(struct dir *);
 extern char *getpath(struct dir *, char *);
 /* settings.c */
-extern int settingsCli(int, char **);
 extern int settingsWin(void);
 /* calc.c */
 extern struct dir *showCalc(char *);
@@ -157,4 +172,6 @@ extern struct dir *showDelete(struct dir *);
 extern void addExclude(char *);
 extern int addExcludeFile(char *);
 extern int matchExclude(char *);
-
+/* export.c */
+extern void exportFile(char *, struct dir *);
+extern struct dir *importFile(char *);

src/settings.c

@@ -26,60 +26,6 @@
 #include "ncdu.h"
 
 
-int settingsCli(int argc, char **argv) {
-  int i, j;
-  char gotdir = 0;
-
- /* load defaults */
-  memset(sdir, 0, PATH_MAX);
-  getcwd(sdir, PATH_MAX);
-  sflags = 0;
-  sdelay = 100;
-  bflags = BF_SIZE | BF_DESC;
-
- /* read from commandline */
-  for(i=1; i<argc; i++) {
-    if(argv[i][0] == '-') {
-      if(argv[i][1] == 'X' || strcmp(argv[i], "--exclude-from") == 0 || strcmp(argv[i], "--exclude") == 0) {
-        if(i+1 >= argc) {
-          printf("Option %s requires an argument\n", argv[i]);
-          exit(1);
-        }
-        if(strcmp(argv[i], "--exclude") == 0)
-          addExclude(argv[++i]);
-        else if(addExcludeFile(argv[++i])) {
-          printf("Can't open %s: %s\n", argv[i], strerror(errno));
-          exit(1);
-        }
-        continue;
-      }
-      for(j=1; j < strlen(argv[i]); j++)
-        switch(argv[i][j]) {
-          case 'x': sflags |= SF_SMFS; break;
-          case 'q': sdelay = 2000;     break;
-          case '?':
-          case 'h':
-            printf("ncdu [-ahvx] [dir]\n\n");
-            printf("  -h    This help message\n");
-            printf("  -q x  Set the refresh interval in seconds\n");
-            printf("  -v    Print version\n");
-            printf("  -x    Same filesystem\n");
-            exit(0);
-          case 'v':
-            printf("ncdu %s\n", PACKAGE_VERSION);
-            exit(0);  
-          default:
-            printf("Unknown option: -%c\n", argv[i][j]);
-            exit(1);
-        }
-    } else {
-      strcpy(sdir, argv[i]);
-      gotdir = 1;
-    }
-  }
-  return(gotdir ? 0 : 1);
-}
-
 int settingsGet(void) {
   WINDOW *set;
   FORM *setf;
@@ -87,6 +33,10 @@ int settingsGet(void) {
   int w, h, cx, cy, i, j, ch;
   int fw, fh, fy, fx, fnrow, fnbuf;
   char tmp[10], *buf = "", rst = 0;
+
+  if(!sdir[0])
+    getcwd(sdir, PATH_MAX);
+
   erase();
   refresh();
                     /*  h,  w, y,  x  */