New API function iso_file_add_external_filter()

with public parameter class IsoExternalFilterCommand
allows to use child processes as external content filter for IsoFile objects.

Makefile.am

@@ -43,6 +43,7 @@ libisofs_libisofs_la_SOURCES = \
 	libisofs/filter.h \
 	libisofs/filter.c \
 	libisofs/filters/xor_encrypt.c \
+	libisofs/filters/external.c \
 	libisofs/util.h \
 	libisofs/util.c \
 	libisofs/util_rbtree.c \

libisofs/aaip_0_2.c

@@ -58,8 +58,8 @@
 static char Aaip_namespace_textS[][16]= {"", "", "system.", "user.", "isofs.",
                                          "trusted.", "security."};
 
-/* maximum expansion:  "user.aaipXY_" */
+/* maximum expansion:  "security." */
-#define Aaip_max_name_expansioN  12
+#define Aaip_max_name_expansioN  9
 
 /* --------------------------------- Encoder ---------------------------- */
 

libisofs/filesrc.c

@@ -36,6 +36,13 @@ int iso_file_src_cmp(const void *n1, const void *n2)
     iso_stream_get_id(f1->stream, &fs_id1, &dev_id1, &ino_id1);
     iso_stream_get_id(f2->stream, &fs_id2, &dev_id2, &ino_id2);
 
+#ifdef Libisofs_file_src_cmp_non_zerO
+    if (fs_id1 == 0 && dev_id1 == 0 && ino_id1 == 0)
+        return -1;
+    if (fs_id2 == 0 && dev_id2 == 0 && ino_id2 == 0)
+        return 1;
+#endif
+
     if (fs_id1 < fs_id2) {
         return -1;
     } else if (fs_id1 > fs_id2) {

libisofs/filter.h

@@ -12,8 +12,16 @@
  * Definitions of filters.
  */
 
+
 /* dev_id for stream identification */
-#define XOR_ENCRYPT_DEV_ID      1
+
+/* libisofs/filters/xor_encrypt.c */
+#define XOR_ENCRYPT_DEV_ID           1
+
+/* libisofs/filters/external.c */
+#define ISO_FILTER_EXTERNAL_DEV_ID   2
+
+
 
 typedef struct filter_context FilterContext;
 

libisofs/filters/external.c

@@ -0,0 +1,514 @@
+/*
+ * Copyright (c) 2009 Thomas Schmitt
+ * 
+ * This file is part of the libisofs project; you can redistribute it and/or 
+ * modify it under the terms of the GNU General Public License version 2 as 
+ * published by the Free Software Foundation. See COPYING file for details.
+ *
+ * It implements a filter facility which can pipe a IsoStream into an external
+ * process, read its output and forward it as IsoStream output to an IsoFile.
+ * The external processes get started according to an IsoExternalFilterCommand
+ * which is described in libisofs.h.
+ * 
+ */
+
+#include "../libisofs.h"
+#include "../filter.h"
+#include "../fsource.h"
+
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <signal.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <errno.h>
+
+
+/*
+ * A filter that starts an external process and uses its stdin and stdout
+ * for classical pipe filtering.
+ */
+
+
+/*
+ * Individual runtime properties exist only as long as stream is opened.
+ */
+typedef struct
+{
+    int send_fd;
+    int recv_fd;
+    pid_t pid;
+    int eof;
+} ExternalFilterRuntime;
+
+
+/*
+ * The data payload of an individual IsoStream from External Filter
+ */
+typedef struct
+{
+    ino_t id;
+
+    IsoStream *orig;
+
+    IsoExternalFilterCommand *cmd;
+
+    off_t size; /* -1 means that the size is unknown yet */
+
+    ExternalFilterRuntime *running; /* is non-NULL when open */
+
+} ExternalFilterStreamData;
+
+
+/* Each individual ExternalFilterStreamData needs a unique id number. */
+/* >>> This is very suboptimal:
+       The counter can rollover.
+*/
+static ino_t extf_ino_id = 0;
+
+
+/*
+ * Methods for the IsoStreamIface of an External Filter object.
+ */
+
+
+/*
+ * @param flag  bit0= do not run .get_size() if size is < 0
+ */
+static
+int extf_stream_open_flag(IsoStream *stream, int flag)
+{
+    ExternalFilterStreamData *data;
+    ExternalFilterRuntime *running = NULL;
+    pid_t child_pid;
+    int send_pipe[2], recv_pipe[2], ret, stream_open = 0;
+
+    send_pipe[0] = send_pipe[1] = recv_pipe[0] = recv_pipe[1] = -1;
+
+    if (stream == NULL) {
+        return ISO_NULL_POINTER;
+    }
+    data = (ExternalFilterStreamData*)stream->data;
+
+    if (data->running != NULL) {
+        return ISO_FILE_ALREADY_OPENED;
+    }
+    if (data->size < 0 && !(flag & 1)) {
+        /* Do the size determination run now, so that the size gets cached
+           and .get_size() will not fail on an opened stream.
+         */
+        stream->class->get_size(stream);
+    }
+    ret = iso_stream_open(data->orig);
+    if (ret < 0) {
+        return ret;
+    }
+    stream_open = 1;
+
+    ret = pipe(send_pipe);
+    if (ret == -1) {
+        ret = ISO_OUT_OF_MEM;
+        goto parent_failed;
+    }
+    ret = pipe(recv_pipe);
+    if (ret == -1) {
+        ret = ISO_OUT_OF_MEM;
+        goto parent_failed;
+    }
+
+    child_pid= fork();
+    if (child_pid == -1) {
+        ret = ISO_DATA_SOURCE_FATAL;
+        goto parent_failed;
+    }
+
+    if (child_pid != 0) {
+        /* parent */
+
+        running = calloc(sizeof(ExternalFilterRuntime), 1);
+        if (running == NULL) {
+            ret = ISO_OUT_OF_MEM;
+            goto parent_failed;
+        }
+        running->send_fd = send_pipe[1];
+        running->recv_fd = recv_pipe[0];
+        running->pid = child_pid;
+        running->eof = 0;
+        data->running = running;
+
+        /* Give up the child-side pipe ends */
+        close(send_pipe[0]);
+        close(recv_pipe[1]);
+
+        /* >>> ??? should one replace non-blocking read() by select () ? */
+
+        /* Make filter outlet non-blocking */
+        ret = fcntl(recv_pipe[0], F_GETFL);
+        if (ret != -1) {
+            ret |= O_NONBLOCK;
+            fcntl(recv_pipe[0], F_SETFL, ret);
+        }
+
+        return 1;
+    }
+
+    /* child */
+
+    /* Give up the parent-side pipe ends */
+    close(send_pipe[1]);
+    close(recv_pipe[0]);
+
+    /* attach pipe ends to stdin and stdout */;
+    close(0);
+    ret = dup2(send_pipe[0], 0);
+    if (ret == -1) {
+        goto child_failed;
+    }
+    close(1);
+    ret = dup2(recv_pipe[1], 1);
+    if (ret == -1) {
+        goto child_failed;
+    }
+
+    /* Self conversion into external program */
+    execv(data->cmd->path, data->cmd->argv); /* should never come back */
+
+child_failed:;
+    fprintf(stderr,"--- execution of external filter command failed:\n");
+    fprintf(stderr,"    %s\n", data->cmd->path);
+    exit(127);
+
+parent_failed:;
+    if (stream_open)
+        iso_stream_close(data->orig);
+    if(send_pipe[0] != -1)
+        close(send_pipe[0]);
+    if(send_pipe[1] != -1)
+        close(send_pipe[1]);
+    if(recv_pipe[0] != -1)
+        close(recv_pipe[0]);
+    if(recv_pipe[1] != -1)
+        close(recv_pipe[1]);
+    return ret;
+}
+
+
+static
+int extf_stream_open(IsoStream *stream)
+{
+    return extf_stream_open_flag(stream, 0);
+}
+
+
+static
+int extf_stream_close(IsoStream *stream)
+{
+    int ret, status;
+    ExternalFilterStreamData *data;
+    if (stream == NULL) {
+        return ISO_NULL_POINTER;
+    }
+    data = stream->data;
+
+    if (data->running != NULL) {
+        if(data->running->recv_fd != -1)
+            close(data->running->recv_fd);
+        if(data->running->send_fd != -1)
+            close(data->running->send_fd);
+
+        ret = waitpid(data->running->pid, &status, WNOHANG);
+        if (ret == -1) {
+            kill(data->running->pid, SIGKILL);
+            waitpid(data->running->pid, &status, 0);
+        }
+
+        free(data->running);
+        data->running = NULL;
+    }
+    return iso_stream_close(data->orig);
+}
+
+
+static
+int extf_stream_read(IsoStream *stream, void *buf, size_t desired)
+{
+    int ret, in_done = 0, blocking = 0;
+    ExternalFilterStreamData *data;
+    uint8_t pipebuf[2048]; /* keep this small, not to clog the input */
+    size_t fill = 0;
+
+    if (stream == NULL) {
+        return ISO_NULL_POINTER;
+    }
+    data = stream->data;
+    if (data->running == NULL) {
+        return ISO_FILE_NOT_OPENED;
+    }
+    if (data->running->eof) {
+        return 0;
+    }
+
+    while (1) {
+
+        /* Try to read desired amount from filter */;
+        while (1) {
+
+            /* >>> ??? should one replace non-blocking read() by select () ? */
+
+            ret = read(data->running->recv_fd, ((char *) buf) + fill,
+                       desired - fill);
+            if (ret == -1) {
+                if (errno == EAGAIN)
+        break;
+                return ISO_FILE_READ_ERROR;
+            }
+            fill += ret;
+            if (ret == 0) {
+                data->running->eof = 1;
+            }
+            if (ret == 0 || fill >= desired) {
+                return fill;
+            }
+        }
+
+        if (in_done) {
+
+            /* >>> ??? should one replace non-blocking read() by select () ? */
+
+            /* Make filter outlet blocking */
+            ret = fcntl(data->running->recv_fd, F_GETFL);
+            if (ret != -1) {
+                ret &= ~O_NONBLOCK;
+                fcntl(data->running->recv_fd, F_SETFL, ret);
+            }
+            blocking = 1;
+            usleep(1000); /* just in case it is still non-blocking */
+    continue;
+        }
+        ret = iso_stream_read(data->orig, pipebuf, sizeof(pipebuf));
+        if (ret < 0) {
+            return ret;
+        }
+        if (ret == 0) {
+            in_done = 1;
+            close(data->running->send_fd); /* Tell the filter: it is over */
+            data->running->send_fd = -1;
+        } else {
+            ret = write(data->running->send_fd, pipebuf, ret);
+            if (ret == -1) {
+                /* From the view of the caller it _is_ a read error */
+                return ISO_FILE_READ_ERROR;
+            }
+        }
+    }
+    return ISO_FILE_READ_ERROR; /* should never be hit */
+}
+
+
+static
+off_t extf_stream_get_size(IsoStream *stream)
+{
+    int ret, ret_close;
+    off_t count = 0;
+    ExternalFilterStreamData *data;
+    char buf[64 * 1024];
+    size_t bufsize = 64 * 1024;
+
+    if (stream == NULL) {
+        return ISO_NULL_POINTER;
+    }
+    data = stream->data;
+
+    if (data->size >= 0) {
+        return data->size;
+    }
+
+    /* Run filter command and count output bytes */
+    ret = extf_stream_open_flag(stream, 1);
+    if (ret < 0) {
+        return ret;
+    }
+    while (1) {
+        ret = extf_stream_read(stream, buf, bufsize);
+        if (ret <= 0)
+            break;
+        count += ret;
+    }
+    ret_close = extf_stream_close(stream);
+    if (ret < 0)
+        return ret;
+    if (ret_close < 0)
+        return ret_close;
+
+    data->size = count;
+    return count;
+}
+
+
+static
+int extf_stream_is_repeatable(IsoStream *stream)
+{
+    /* Only repeatable streams are accepted as orig */
+    return 1;
+}
+
+
+static
+void extf_stream_get_id(IsoStream *stream, unsigned int *fs_id, 
+                        dev_t *dev_id, ino_t *ino_id)
+{
+    ExternalFilterStreamData *data;
+
+    data = stream->data;
+    *fs_id = ISO_FILTER_FS_ID;
+    *dev_id = ISO_FILTER_EXTERNAL_DEV_ID;
+    *ino_id = data->id;
+}
+
+
+static
+void extf_stream_free(IsoStream *stream)
+{
+    ExternalFilterStreamData *data;
+
+    if (stream == NULL) {
+        return;
+    }
+    data = stream->data;
+    iso_stream_unref(data->orig);
+    if (data->cmd->refcount > 0)
+        data->cmd->refcount--;
+    free(data);
+}
+
+
+IsoStreamIface extf_stream_class = {
+    0,
+    "extf",
+    extf_stream_open,
+    extf_stream_close,
+    extf_stream_get_size,
+    extf_stream_read,
+    extf_stream_is_repeatable,
+    extf_stream_get_id,
+    extf_stream_free
+};
+
+
+static
+void extf_filter_free(FilterContext *filter)
+{
+    /* no data are allocated */;
+}
+
+
+/* To be called by iso_file_add_filter().
+ * The FilterContext input parameter is not furtherly needed for the 
+ * emerging IsoStream.
+ */
+static
+int extf_filter_get_filter(FilterContext *filter, IsoStream *original, 
+                  IsoStream **filtered)
+{
+    IsoStream *str;
+    ExternalFilterStreamData *data;
+    IsoExternalFilterCommand *cmd;
+
+    if (filter == NULL || original == NULL || filtered == NULL) {
+        return ISO_NULL_POINTER;
+    }
+    cmd = (IsoExternalFilterCommand *) filter->data;
+    if (cmd->refcount + 1 <= 0) {
+        return ISO_EXTF_TOO_OFTEN;
+    }
+
+    str = malloc(sizeof(IsoStream));
+    if (str == NULL) {
+        return ISO_OUT_OF_MEM;
+    }
+    data = malloc(sizeof(ExternalFilterStreamData));
+    if (str == NULL) {
+        free(str);
+        return ISO_OUT_OF_MEM;
+    }
+
+
+    /* These data items are not owned by this filter object */
+    data->id = ++extf_ino_id;
+    data->orig = original;
+    data->cmd = cmd;
+    data->size = -1;
+    data->running = NULL;
+
+    /* get reference to the source */
+    iso_stream_ref(data->orig);
+
+    str->refcount = 1;
+    str->data = data;
+    str->class = &extf_stream_class;
+
+    *filtered = str;
+
+    cmd->refcount++;
+    return ISO_SUCCESS;
+}
+
+
+/* Produce a parameter object suitable for iso_file_add_filter().
+ * It may be disposed by free() after all those calls are made.
+ *
+ * This is an internal call of libisofs to be used by an API call that
+ * attaches an IsoExternalFilterCommand to one or more IsoFile objects.
+ * See libisofs.h for IsoExternalFilterCommand.
+ */
+static
+int extf_create_context(IsoExternalFilterCommand *cmd,
+                        FilterContext **filter, int flag)
+{
+    FilterContext *f;
+    
+    *filter = f = calloc(1, sizeof(FilterContext));
+    if (f == NULL) {
+        return ISO_OUT_OF_MEM;
+    }
+    f->refcount = 1;
+    f->version = 0;
+    f->data = cmd;
+    f->free = extf_filter_free;
+    f->get_filter = extf_filter_get_filter;
+    return ISO_SUCCESS;
+}
+
+
+/*
+ * A function which adds a filter to an IsoFile shall create a temporary
+ * FilterContext by iso_extf_create_context(), use it in one or more calls
+ * of filter.c:iso_file_add_filter() and finally dispose it by free().
+ */
+
+int iso_file_add_external_filter(IsoFile *file, IsoExternalFilterCommand *cmd,
+                                 int flag)
+{
+    int ret;
+    FilterContext *f = NULL;
+    IsoStream *stream;
+
+    ret = extf_create_context(cmd, &f, 0);
+    if (ret < 0) {
+        return ret;
+    }
+    ret = iso_file_add_filter(file, f, 0);
+    free(f);
+    if (ret < 0) {
+        return ret;
+    }
+    /* Run a full filter process getsize so that the size is cached */
+    stream = iso_file_get_stream(file);
+    ret = iso_stream_get_size(stream);
+    if (ret < 0) {
+        return ret;
+    }
+    return ISO_SUCCESS;
+}
+

libisofs/libisofs.h

@@ -778,6 +778,7 @@ struct IsoStream_Iface
      * "fsrc" -> Read from file source
      * "mem " -> Read from memory
      * "boot" -> Boot catalog
+     * "extf" -> External filter program
      * "user" -> User supplied stream
      */
     char type[4];
@@ -872,6 +873,7 @@ struct iso_stream
     void *data;
 };
 
+
 /**
  * Initialize libisofs. Before any usage of the library you must either call
  * this function or iso_init_with_flag().
@@ -2837,10 +2839,11 @@ dev_t iso_special_get_dev(IsoSpecial *special);
 /**
  * Get the IsoStream that represents the contents of the given IsoFile.
  *
- * If you open() the stream, it should be close() before image generation.
+ * If you iso_stream_open() the stream, iso_stream_close() it before
+ * image generation begins.
  *
  * @return
- *      The IsoStream. No extra ref is added, so the IsoStream belong to the
+ *      The IsoStream. No extra ref is added, so the IsoStream belongs to the
  *      IsoFile, and it may be freed together with it. Add your own ref with
  *      iso_stream_ref() if you need it.
  *
@@ -4183,7 +4186,7 @@ void iso_stream_get_id(IsoStream *stream, unsigned int *fs_id, dev_t *dev_id,
 #define ISO_CHARSET_CONV_ERROR          0xE830FF00
 
 /**
- * Too much files to mangle, i.e. we cannot guarantee unique file names
+ * Too many files to mangle, i.e. we cannot guarantee unique file names
  * (FAILURE,HIGH, -257)
  */
 #define ISO_MANGLE_TOO_MUCH_FILES       0xE830FEFF
@@ -4278,9 +4281,14 @@ void iso_stream_get_id(IsoStream *stream, unsigned int *fs_id, dev_t *dev_id,
 #define ISO_AAIP_NO_SET_LOCAL     0xE830FEAA
 
 /** Unallowed attempt to set an xattr with non-userspace name
-                                                       (FAILURE, HIGH, -343) */
+                                                    (FAILURE, HIGH, -343) */
 #define ISO_AAIP_NON_USER_NAME    0xE830FEA9
 
+/* ts A90325 */
+/** Too many references on a single IsoExternalFilterCommand
+                                                    (FAILURE, HIGH, -344) */
+#define ISO_EXTF_TOO_OFTEN        0xE830FEA8
+
 
 /* --------------------------------- AAIP --------------------------------- */
 
@@ -4622,6 +4630,76 @@ int iso_local_set_attrs(char *disk_path, size_t num_attrs, char **names,
                         size_t *value_lengths, char **values, int flag);
 
 
+/* ------------------------------------------------------------------------- */
+
+/* >>> ts A90325 */
+/**
+ * Representation of an external program that shall serve as filter for
+ * an IsoStream. This object may be shared among many IsoStream objects.
+ * It is to be created and disposed by the application.
+ *
+ * The filter will act as proxy between the original IsoStream of an IsoFile.
+ * Up to completed image generation it will be run at least twice: 
+ * for IsoStream.class.get_size() and for .open() with subsequent .read().
+ * So the original IsoStream has to return 1 by its .class.is_repeatable().
+ * The filter program has to be repeateable too. I.e. it must produce the same
+ * output on the same input.
+ *
+ * @since 0.6.18
+ */
+struct iso_external_filter_command
+{
+    /* Will indicate future extensions. It has to be 0 for now. */
+    int version;
+
+    /* Tells how many IsoStream objects depend on this command object.
+     * One may only dispose an IsoExternalFilterCommand when this count is 0.
+     * Initially this value has to be 0.
+     */
+    int refcount;
+
+    /* Absolute local filesystem path to the executable program. */
+    char *path;
+
+    /* Tells the number of arguments. */
+    int argc;
+
+    /* NULL terminated list suitable for system call execv(3).
+     * I.e. argv[0] points to the alleged program name,
+     *      argv[1] to argv[argc] point to program arguments (if argc > 0)
+     *      argv[argc+1] is NULL
+     */
+    char **argv;
+
+};
+
+typedef struct iso_external_filter_command IsoExternalFilterCommand;
+
+/* ts A90326 */
+/**
+ * Install an external filter command on top of the content stream of a data
+ * file. The filter process must be repeatable. It will be run once by this
+ * call in order to cache the output size.
+ * iso_file_get_stream() will return the filter stream.
+ * iso_stream_get_size() will return the cached size of the filtered data,
+ * iso_stream_open()     will start again the external filter process,
+ * iso_stream_close()    will kill it,
+ * iso_stream_read()     will return filtered data.
+ * @param file
+ *      The data file node which shall show filtered content.
+ * @param cmd
+ *      The external program and its arguments which shall do the filtering.
+ * @param flag
+ *      Bitfield for control purposes, unused yet, submit 0.
+ * @return
+ *      1 on success, <0 on error
+ *
+ * @since 0.6.18
+ */
+int iso_file_add_external_filter(IsoFile *file, IsoExternalFilterCommand *cmd,
+                                 int flag);
+
+
 /* ------------------------------------------------------------------------- */
 
 #ifdef LIBISOFS_WITHOUT_LIBBURN
@@ -4834,6 +4912,12 @@ struct burn_source {
 #define Libisofs_setlocale_in_iniT yes
 
 
+/* Stabilization: Trying to avoid the risk of losing file content by duplicate
+                  inodes. iso_file_src_cmp() shall compare sizes too.
+*/
+#define Libisofs_file_src_cmp_sizE yes
+
+
 /* ---------------------------- Experiments ---------------------------- */
 
 
@@ -4843,11 +4927,6 @@ struct burn_source {
 #define Libisofs_new_fs_image_inO yes
 
 
-/* Experiment: Trying to avoid the risk of losing file content by duplicate
-               inodes. iso_file_src_cmp() shall compare sizes too.
-*/
-#define Libisofs_file_src_cmp_sizE yes
-
 
 /* Experiment: Revoke Ticket 144, use data file LBAs again.
                (will work only if not Libisofs_new_fs_image_inO
@@ -4871,4 +4950,12 @@ struct burn_source {
 */
 #define Libisofs_with_iso_iconV yes
 
+/* Experiment: Regarding (fs_id == 0 && dev_id == 0 && ino_id == 0)
+                  as always unique.
+   LOOKS DANGEROUS: iso_rbtree_insert() seems to need equality
+
+ #ifdef Libisofs_file_src_cmp_non_zerO yes
+*/
+
+
 #endif /*LIBISO_LIBISOFS_H_*/

libisofs/messages.c

@@ -247,6 +247,8 @@ const char *iso_error_to_msg(int errcode)
         return "Error with attaching ACL or xattr to local file";
     case ISO_AAIP_NON_USER_NAME:
         return "Unallowed attempt to set an xattr with non-userspace name";
+    case ISO_EXTF_TOO_OFTEN:
+        return "Too many references on a single external filter command";
     default:
         return "Unknown error";
     }