diff --git a/Makefile.am b/Makefile.am index de694a9..e7e5552 100644 --- a/Makefile.am +++ b/Makefile.am @@ -43,6 +43,7 @@ libisofs_libisofs_la_SOURCES = \ libisofs/filter.h \ libisofs/filter.c \ libisofs/filters/xor_encrypt.c \ + libisofs/filters/external.c \ libisofs/util.h \ libisofs/util.c \ libisofs/util_rbtree.c \ diff --git a/libisofs/aaip_0_2.c b/libisofs/aaip_0_2.c index 8a3b18f..d28a1a7 100644 --- a/libisofs/aaip_0_2.c +++ b/libisofs/aaip_0_2.c @@ -58,8 +58,8 @@ static char Aaip_namespace_textS[][16]= {"", "", "system.", "user.", "isofs.", "trusted.", "security."}; -/* maximum expansion: "user.aaipXY_" */ -#define Aaip_max_name_expansioN 12 +/* maximum expansion: "security." */ +#define Aaip_max_name_expansioN 9 /* --------------------------------- Encoder ---------------------------- */ diff --git a/libisofs/filesrc.c b/libisofs/filesrc.c index 2d71b37..fdffd28 100644 --- a/libisofs/filesrc.c +++ b/libisofs/filesrc.c @@ -36,6 +36,13 @@ int iso_file_src_cmp(const void *n1, const void *n2) iso_stream_get_id(f1->stream, &fs_id1, &dev_id1, &ino_id1); iso_stream_get_id(f2->stream, &fs_id2, &dev_id2, &ino_id2); +#ifdef Libisofs_file_src_cmp_non_zerO + if (fs_id1 == 0 && dev_id1 == 0 && ino_id1 == 0) + return -1; + if (fs_id2 == 0 && dev_id2 == 0 && ino_id2 == 0) + return 1; +#endif + if (fs_id1 < fs_id2) { return -1; } else if (fs_id1 > fs_id2) { diff --git a/libisofs/filter.h b/libisofs/filter.h index f711fe3..401f7f8 100644 --- a/libisofs/filter.h +++ b/libisofs/filter.h @@ -12,8 +12,16 @@ * Definitions of filters. */ + /* dev_id for stream identification */ -#define XOR_ENCRYPT_DEV_ID 1 + +/* libisofs/filters/xor_encrypt.c */ +#define XOR_ENCRYPT_DEV_ID 1 + +/* libisofs/filters/external.c */ +#define ISO_FILTER_EXTERNAL_DEV_ID 2 + + typedef struct filter_context FilterContext; diff --git a/libisofs/filters/external.c b/libisofs/filters/external.c new file mode 100644 index 0000000..706a4aa --- /dev/null +++ b/libisofs/filters/external.c @@ -0,0 +1,514 @@ +/* + * Copyright (c) 2009 Thomas Schmitt + * + * This file is part of the libisofs project; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. See COPYING file for details. + * + * It implements a filter facility which can pipe a IsoStream into an external + * process, read its output and forward it as IsoStream output to an IsoFile. + * The external processes get started according to an IsoExternalFilterCommand + * which is described in libisofs.h. + * + */ + +#include "../libisofs.h" +#include "../filter.h" +#include "../fsource.h" + +#include +#include +#include +#include +#include +#include +#include + + +/* + * A filter that starts an external process and uses its stdin and stdout + * for classical pipe filtering. + */ + + +/* + * Individual runtime properties exist only as long as stream is opened. + */ +typedef struct +{ + int send_fd; + int recv_fd; + pid_t pid; + int eof; +} ExternalFilterRuntime; + + +/* + * The data payload of an individual IsoStream from External Filter + */ +typedef struct +{ + ino_t id; + + IsoStream *orig; + + IsoExternalFilterCommand *cmd; + + off_t size; /* -1 means that the size is unknown yet */ + + ExternalFilterRuntime *running; /* is non-NULL when open */ + +} ExternalFilterStreamData; + + +/* Each individual ExternalFilterStreamData needs a unique id number. */ +/* >>> This is very suboptimal: + The counter can rollover. +*/ +static ino_t extf_ino_id = 0; + + +/* + * Methods for the IsoStreamIface of an External Filter object. + */ + + +/* + * @param flag bit0= do not run .get_size() if size is < 0 + */ +static +int extf_stream_open_flag(IsoStream *stream, int flag) +{ + ExternalFilterStreamData *data; + ExternalFilterRuntime *running = NULL; + pid_t child_pid; + int send_pipe[2], recv_pipe[2], ret, stream_open = 0; + + send_pipe[0] = send_pipe[1] = recv_pipe[0] = recv_pipe[1] = -1; + + if (stream == NULL) { + return ISO_NULL_POINTER; + } + data = (ExternalFilterStreamData*)stream->data; + + if (data->running != NULL) { + return ISO_FILE_ALREADY_OPENED; + } + if (data->size < 0 && !(flag & 1)) { + /* Do the size determination run now, so that the size gets cached + and .get_size() will not fail on an opened stream. + */ + stream->class->get_size(stream); + } + ret = iso_stream_open(data->orig); + if (ret < 0) { + return ret; + } + stream_open = 1; + + ret = pipe(send_pipe); + if (ret == -1) { + ret = ISO_OUT_OF_MEM; + goto parent_failed; + } + ret = pipe(recv_pipe); + if (ret == -1) { + ret = ISO_OUT_OF_MEM; + goto parent_failed; + } + + child_pid= fork(); + if (child_pid == -1) { + ret = ISO_DATA_SOURCE_FATAL; + goto parent_failed; + } + + if (child_pid != 0) { + /* parent */ + + running = calloc(sizeof(ExternalFilterRuntime), 1); + if (running == NULL) { + ret = ISO_OUT_OF_MEM; + goto parent_failed; + } + running->send_fd = send_pipe[1]; + running->recv_fd = recv_pipe[0]; + running->pid = child_pid; + running->eof = 0; + data->running = running; + + /* Give up the child-side pipe ends */ + close(send_pipe[0]); + close(recv_pipe[1]); + + /* >>> ??? should one replace non-blocking read() by select () ? */ + + /* Make filter outlet non-blocking */ + ret = fcntl(recv_pipe[0], F_GETFL); + if (ret != -1) { + ret |= O_NONBLOCK; + fcntl(recv_pipe[0], F_SETFL, ret); + } + + return 1; + } + + /* child */ + + /* Give up the parent-side pipe ends */ + close(send_pipe[1]); + close(recv_pipe[0]); + + /* attach pipe ends to stdin and stdout */; + close(0); + ret = dup2(send_pipe[0], 0); + if (ret == -1) { + goto child_failed; + } + close(1); + ret = dup2(recv_pipe[1], 1); + if (ret == -1) { + goto child_failed; + } + + /* Self conversion into external program */ + execv(data->cmd->path, data->cmd->argv); /* should never come back */ + +child_failed:; + fprintf(stderr,"--- execution of external filter command failed:\n"); + fprintf(stderr," %s\n", data->cmd->path); + exit(127); + +parent_failed:; + if (stream_open) + iso_stream_close(data->orig); + if(send_pipe[0] != -1) + close(send_pipe[0]); + if(send_pipe[1] != -1) + close(send_pipe[1]); + if(recv_pipe[0] != -1) + close(recv_pipe[0]); + if(recv_pipe[1] != -1) + close(recv_pipe[1]); + return ret; +} + + +static +int extf_stream_open(IsoStream *stream) +{ + return extf_stream_open_flag(stream, 0); +} + + +static +int extf_stream_close(IsoStream *stream) +{ + int ret, status; + ExternalFilterStreamData *data; + if (stream == NULL) { + return ISO_NULL_POINTER; + } + data = stream->data; + + if (data->running != NULL) { + if(data->running->recv_fd != -1) + close(data->running->recv_fd); + if(data->running->send_fd != -1) + close(data->running->send_fd); + + ret = waitpid(data->running->pid, &status, WNOHANG); + if (ret == -1) { + kill(data->running->pid, SIGKILL); + waitpid(data->running->pid, &status, 0); + } + + free(data->running); + data->running = NULL; + } + return iso_stream_close(data->orig); +} + + +static +int extf_stream_read(IsoStream *stream, void *buf, size_t desired) +{ + int ret, in_done = 0, blocking = 0; + ExternalFilterStreamData *data; + uint8_t pipebuf[2048]; /* keep this small, not to clog the input */ + size_t fill = 0; + + if (stream == NULL) { + return ISO_NULL_POINTER; + } + data = stream->data; + if (data->running == NULL) { + return ISO_FILE_NOT_OPENED; + } + if (data->running->eof) { + return 0; + } + + while (1) { + + /* Try to read desired amount from filter */; + while (1) { + + /* >>> ??? should one replace non-blocking read() by select () ? */ + + ret = read(data->running->recv_fd, ((char *) buf) + fill, + desired - fill); + if (ret == -1) { + if (errno == EAGAIN) + break; + return ISO_FILE_READ_ERROR; + } + fill += ret; + if (ret == 0) { + data->running->eof = 1; + } + if (ret == 0 || fill >= desired) { + return fill; + } + } + + if (in_done) { + + /* >>> ??? should one replace non-blocking read() by select () ? */ + + /* Make filter outlet blocking */ + ret = fcntl(data->running->recv_fd, F_GETFL); + if (ret != -1) { + ret &= ~O_NONBLOCK; + fcntl(data->running->recv_fd, F_SETFL, ret); + } + blocking = 1; + usleep(1000); /* just in case it is still non-blocking */ + continue; + } + ret = iso_stream_read(data->orig, pipebuf, sizeof(pipebuf)); + if (ret < 0) { + return ret; + } + if (ret == 0) { + in_done = 1; + close(data->running->send_fd); /* Tell the filter: it is over */ + data->running->send_fd = -1; + } else { + ret = write(data->running->send_fd, pipebuf, ret); + if (ret == -1) { + /* From the view of the caller it _is_ a read error */ + return ISO_FILE_READ_ERROR; + } + } + } + return ISO_FILE_READ_ERROR; /* should never be hit */ +} + + +static +off_t extf_stream_get_size(IsoStream *stream) +{ + int ret, ret_close; + off_t count = 0; + ExternalFilterStreamData *data; + char buf[64 * 1024]; + size_t bufsize = 64 * 1024; + + if (stream == NULL) { + return ISO_NULL_POINTER; + } + data = stream->data; + + if (data->size >= 0) { + return data->size; + } + + /* Run filter command and count output bytes */ + ret = extf_stream_open_flag(stream, 1); + if (ret < 0) { + return ret; + } + while (1) { + ret = extf_stream_read(stream, buf, bufsize); + if (ret <= 0) + break; + count += ret; + } + ret_close = extf_stream_close(stream); + if (ret < 0) + return ret; + if (ret_close < 0) + return ret_close; + + data->size = count; + return count; +} + + +static +int extf_stream_is_repeatable(IsoStream *stream) +{ + /* Only repeatable streams are accepted as orig */ + return 1; +} + + +static +void extf_stream_get_id(IsoStream *stream, unsigned int *fs_id, + dev_t *dev_id, ino_t *ino_id) +{ + ExternalFilterStreamData *data; + + data = stream->data; + *fs_id = ISO_FILTER_FS_ID; + *dev_id = ISO_FILTER_EXTERNAL_DEV_ID; + *ino_id = data->id; +} + + +static +void extf_stream_free(IsoStream *stream) +{ + ExternalFilterStreamData *data; + + if (stream == NULL) { + return; + } + data = stream->data; + iso_stream_unref(data->orig); + if (data->cmd->refcount > 0) + data->cmd->refcount--; + free(data); +} + + +IsoStreamIface extf_stream_class = { + 0, + "extf", + extf_stream_open, + extf_stream_close, + extf_stream_get_size, + extf_stream_read, + extf_stream_is_repeatable, + extf_stream_get_id, + extf_stream_free +}; + + +static +void extf_filter_free(FilterContext *filter) +{ + /* no data are allocated */; +} + + +/* To be called by iso_file_add_filter(). + * The FilterContext input parameter is not furtherly needed for the + * emerging IsoStream. + */ +static +int extf_filter_get_filter(FilterContext *filter, IsoStream *original, + IsoStream **filtered) +{ + IsoStream *str; + ExternalFilterStreamData *data; + IsoExternalFilterCommand *cmd; + + if (filter == NULL || original == NULL || filtered == NULL) { + return ISO_NULL_POINTER; + } + cmd = (IsoExternalFilterCommand *) filter->data; + if (cmd->refcount + 1 <= 0) { + return ISO_EXTF_TOO_OFTEN; + } + + str = malloc(sizeof(IsoStream)); + if (str == NULL) { + return ISO_OUT_OF_MEM; + } + data = malloc(sizeof(ExternalFilterStreamData)); + if (str == NULL) { + free(str); + return ISO_OUT_OF_MEM; + } + + + /* These data items are not owned by this filter object */ + data->id = ++extf_ino_id; + data->orig = original; + data->cmd = cmd; + data->size = -1; + data->running = NULL; + + /* get reference to the source */ + iso_stream_ref(data->orig); + + str->refcount = 1; + str->data = data; + str->class = &extf_stream_class; + + *filtered = str; + + cmd->refcount++; + return ISO_SUCCESS; +} + + +/* Produce a parameter object suitable for iso_file_add_filter(). + * It may be disposed by free() after all those calls are made. + * + * This is an internal call of libisofs to be used by an API call that + * attaches an IsoExternalFilterCommand to one or more IsoFile objects. + * See libisofs.h for IsoExternalFilterCommand. + */ +static +int extf_create_context(IsoExternalFilterCommand *cmd, + FilterContext **filter, int flag) +{ + FilterContext *f; + + *filter = f = calloc(1, sizeof(FilterContext)); + if (f == NULL) { + return ISO_OUT_OF_MEM; + } + f->refcount = 1; + f->version = 0; + f->data = cmd; + f->free = extf_filter_free; + f->get_filter = extf_filter_get_filter; + return ISO_SUCCESS; +} + + +/* + * A function which adds a filter to an IsoFile shall create a temporary + * FilterContext by iso_extf_create_context(), use it in one or more calls + * of filter.c:iso_file_add_filter() and finally dispose it by free(). + */ + +int iso_file_add_external_filter(IsoFile *file, IsoExternalFilterCommand *cmd, + int flag) +{ + int ret; + FilterContext *f = NULL; + IsoStream *stream; + + ret = extf_create_context(cmd, &f, 0); + if (ret < 0) { + return ret; + } + ret = iso_file_add_filter(file, f, 0); + free(f); + if (ret < 0) { + return ret; + } + /* Run a full filter process getsize so that the size is cached */ + stream = iso_file_get_stream(file); + ret = iso_stream_get_size(stream); + if (ret < 0) { + return ret; + } + return ISO_SUCCESS; +} + diff --git a/libisofs/libisofs.h b/libisofs/libisofs.h index 9b4dd0d..72e4747 100644 --- a/libisofs/libisofs.h +++ b/libisofs/libisofs.h @@ -778,6 +778,7 @@ struct IsoStream_Iface * "fsrc" -> Read from file source * "mem " -> Read from memory * "boot" -> Boot catalog + * "extf" -> External filter program * "user" -> User supplied stream */ char type[4]; @@ -872,6 +873,7 @@ struct iso_stream void *data; }; + /** * Initialize libisofs. Before any usage of the library you must either call * this function or iso_init_with_flag(). @@ -2837,10 +2839,11 @@ dev_t iso_special_get_dev(IsoSpecial *special); /** * Get the IsoStream that represents the contents of the given IsoFile. * - * If you open() the stream, it should be close() before image generation. + * If you iso_stream_open() the stream, iso_stream_close() it before + * image generation begins. * * @return - * The IsoStream. No extra ref is added, so the IsoStream belong to the + * The IsoStream. No extra ref is added, so the IsoStream belongs to the * IsoFile, and it may be freed together with it. Add your own ref with * iso_stream_ref() if you need it. * @@ -4183,7 +4186,7 @@ void iso_stream_get_id(IsoStream *stream, unsigned int *fs_id, dev_t *dev_id, #define ISO_CHARSET_CONV_ERROR 0xE830FF00 /** - * Too much files to mangle, i.e. we cannot guarantee unique file names + * Too many files to mangle, i.e. we cannot guarantee unique file names * (FAILURE,HIGH, -257) */ #define ISO_MANGLE_TOO_MUCH_FILES 0xE830FEFF @@ -4278,9 +4281,14 @@ void iso_stream_get_id(IsoStream *stream, unsigned int *fs_id, dev_t *dev_id, #define ISO_AAIP_NO_SET_LOCAL 0xE830FEAA /** Unallowed attempt to set an xattr with non-userspace name - (FAILURE, HIGH, -343) */ + (FAILURE, HIGH, -343) */ #define ISO_AAIP_NON_USER_NAME 0xE830FEA9 +/* ts A90325 */ +/** Too many references on a single IsoExternalFilterCommand + (FAILURE, HIGH, -344) */ +#define ISO_EXTF_TOO_OFTEN 0xE830FEA8 + /* --------------------------------- AAIP --------------------------------- */ @@ -4622,6 +4630,76 @@ int iso_local_set_attrs(char *disk_path, size_t num_attrs, char **names, size_t *value_lengths, char **values, int flag); +/* ------------------------------------------------------------------------- */ + +/* >>> ts A90325 */ +/** + * Representation of an external program that shall serve as filter for + * an IsoStream. This object may be shared among many IsoStream objects. + * It is to be created and disposed by the application. + * + * The filter will act as proxy between the original IsoStream of an IsoFile. + * Up to completed image generation it will be run at least twice: + * for IsoStream.class.get_size() and for .open() with subsequent .read(). + * So the original IsoStream has to return 1 by its .class.is_repeatable(). + * The filter program has to be repeateable too. I.e. it must produce the same + * output on the same input. + * + * @since 0.6.18 + */ +struct iso_external_filter_command +{ + /* Will indicate future extensions. It has to be 0 for now. */ + int version; + + /* Tells how many IsoStream objects depend on this command object. + * One may only dispose an IsoExternalFilterCommand when this count is 0. + * Initially this value has to be 0. + */ + int refcount; + + /* Absolute local filesystem path to the executable program. */ + char *path; + + /* Tells the number of arguments. */ + int argc; + + /* NULL terminated list suitable for system call execv(3). + * I.e. argv[0] points to the alleged program name, + * argv[1] to argv[argc] point to program arguments (if argc > 0) + * argv[argc+1] is NULL + */ + char **argv; + +}; + +typedef struct iso_external_filter_command IsoExternalFilterCommand; + +/* ts A90326 */ +/** + * Install an external filter command on top of the content stream of a data + * file. The filter process must be repeatable. It will be run once by this + * call in order to cache the output size. + * iso_file_get_stream() will return the filter stream. + * iso_stream_get_size() will return the cached size of the filtered data, + * iso_stream_open() will start again the external filter process, + * iso_stream_close() will kill it, + * iso_stream_read() will return filtered data. + * @param file + * The data file node which shall show filtered content. + * @param cmd + * The external program and its arguments which shall do the filtering. + * @param flag + * Bitfield for control purposes, unused yet, submit 0. + * @return + * 1 on success, <0 on error + * + * @since 0.6.18 + */ +int iso_file_add_external_filter(IsoFile *file, IsoExternalFilterCommand *cmd, + int flag); + + /* ------------------------------------------------------------------------- */ #ifdef LIBISOFS_WITHOUT_LIBBURN @@ -4834,6 +4912,12 @@ struct burn_source { #define Libisofs_setlocale_in_iniT yes +/* Stabilization: Trying to avoid the risk of losing file content by duplicate + inodes. iso_file_src_cmp() shall compare sizes too. +*/ +#define Libisofs_file_src_cmp_sizE yes + + /* ---------------------------- Experiments ---------------------------- */ @@ -4843,11 +4927,6 @@ struct burn_source { #define Libisofs_new_fs_image_inO yes -/* Experiment: Trying to avoid the risk of losing file content by duplicate - inodes. iso_file_src_cmp() shall compare sizes too. -*/ -#define Libisofs_file_src_cmp_sizE yes - /* Experiment: Revoke Ticket 144, use data file LBAs again. (will work only if not Libisofs_new_fs_image_inO @@ -4871,4 +4950,12 @@ struct burn_source { */ #define Libisofs_with_iso_iconV yes +/* Experiment: Regarding (fs_id == 0 && dev_id == 0 && ino_id == 0) + as always unique. + LOOKS DANGEROUS: iso_rbtree_insert() seems to need equality + + #ifdef Libisofs_file_src_cmp_non_zerO yes +*/ + + #endif /*LIBISO_LIBISOFS_H_*/ diff --git a/libisofs/messages.c b/libisofs/messages.c index a1d4b3f..05c3641 100644 --- a/libisofs/messages.c +++ b/libisofs/messages.c @@ -247,6 +247,8 @@ const char *iso_error_to_msg(int errcode) return "Error with attaching ACL or xattr to local file"; case ISO_AAIP_NON_USER_NAME: return "Unallowed attempt to set an xattr with non-userspace name"; + case ISO_EXTF_TOO_OFTEN: + return "Too many references on a single external filter command"; default: return "Unknown error"; }