From 8a100f862627f7d825d93ff2e2b7e93b67e000aa Mon Sep 17 00:00:00 2001 From: "Troy D. Hanson" Date: Sun, 26 Feb 2012 19:46:41 -0500 Subject: [PATCH] work in progress --- Makefile.am | 7 +- README | 8 +- TODO | 10 -- configure.ac | 4 +- doc/CREDITS | 8 + doc/LICENSE | 31 ++++ doc/future.txt | 62 +++++++ doc/kvspool-mini.png | Bin 0 -> 1589 bytes kvspool.txt => doc/kvspool.txt | 154 +++++++++++------- kvspool.h => include/kvspool.h | 2 +- .../kvspool_internal.h | 0 src/Makefile.am | 5 + kvspool.c => src/kvspool.c | 0 .../kvspool_attrition.c | 0 kvspoolr.c => src/kvspoolr.c | 2 +- kvspoolw.c => src/kvspoolw.c | 0 tpl.c => src/tpl.c | 0 utils/.gitignore | 2 +- utils/Makefile.am | 26 +-- utils/{kvsp-reset.c => kvsp-rewind.c} | 2 +- 20 files changed, 227 insertions(+), 96 deletions(-) delete mode 100644 TODO create mode 100644 doc/CREDITS create mode 100644 doc/LICENSE create mode 100644 doc/future.txt create mode 100644 doc/kvspool-mini.png rename kvspool.txt => doc/kvspool.txt (79%) rename kvspool.h => include/kvspool.h (98%) rename kvspool_internal.h => include/kvspool_internal.h (100%) create mode 100644 src/Makefile.am rename kvspool.c => src/kvspool.c (100%) rename kvspool_attrition.c => src/kvspool_attrition.c (100%) rename kvspoolr.c => src/kvspoolr.c (99%) rename kvspoolw.c => src/kvspoolw.c (100%) rename tpl.c => src/tpl.c (100%) rename utils/{kvsp-reset.c => kvsp-rewind.c} (96%) diff --git a/Makefile.am b/Makefile.am index 231aaeb..fe58051 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,9 +1,4 @@ -AM_CFLAGS = -fPIC -I./include -lib_LIBRARIES = libkvspool.a -libkvspool_a_SOURCES = kvspool.c kvspoolw.c kvspoolr.c kvspool_attrition.c tpl.c -include_HEADERS = kvspool.h - -SUBDIRS = . utils +SUBDIRS = src utils if HAVE_PYTHON SUBDIRS += kvpy diff --git a/README b/README index abd6cf0..dc82e9a 100644 --- a/README +++ b/README @@ -1,11 +1,7 @@ kvspool data streaming utilities by Troy D. Hanson +Documentation for kvspool is at: + http://troydhanson.github.com/kvspool -Special thanks to: - -Trevor Adams -JT Halbert -Jeff James - diff --git a/TODO b/TODO deleted file mode 100644 index 85f0b05..0000000 --- a/TODO +++ /dev/null @@ -1,10 +0,0 @@ -4. Add documentation/slides -6. Change Python to have object wrapper - -#done -2. Add missing KVJava -3. Move sysutils to their own repo -5. Remove 'base' parameter everywhere -7. zcon and zcontrol should be in a ZeroMQ add-on's repo -8. kvsp-import/export should be subsumed by kvsp-pub/sub -9. put libts in external snippets repo diff --git a/configure.ac b/configure.ac index ca5c140..cbbe1c2 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ AC_PREREQ(2.59) AC_INIT([kvspool], [1.0], [tdh@tkhanson.net]) -AC_CONFIG_SRCDIR(kvspool.c) +AC_CONFIG_SRCDIR(src/kvspool.c) AC_CONFIG_AUX_DIR(config) AC_CONFIG_HEADERS(config/config.h) AM_INIT_AUTOMAKE @@ -24,6 +24,6 @@ AM_CONDITIONAL(HAVE_PYTHON,test "x$pythonexists" = "xyes") AC_CHECK_PROG(PERL,perl,perl) AM_CONDITIONAL(HAVE_PERL,test "x$PERL" != "x") -AC_CONFIG_FILES(Makefile utils/Makefile) +AC_CONFIG_FILES(Makefile src/Makefile utils/Makefile) AC_OUTPUT diff --git a/doc/CREDITS b/doc/CREDITS new file mode 100644 index 0000000..1ae91e9 --- /dev/null +++ b/doc/CREDITS @@ -0,0 +1,8 @@ +kvspool was developed in 2011 by Troy D. Hanson + +Special thanks to: +JHU/APL OTT +Trevor Adams +JT Halbert +Jeff James +Nick Clote diff --git a/doc/LICENSE b/doc/LICENSE new file mode 100644 index 0000000..f2abe0d --- /dev/null +++ b/doc/LICENSE @@ -0,0 +1,31 @@ +LICENSE AND DISCLAIMER + +Copyright (c) 2011 The Johns Hopkins University/Applied Physics Laboratory + +This software was developed at The Johns Hopkins University/Applied Physics +Laboratory (“JHU/APL”) that is the author thereof under the “work made for +hire” provisions of the copyright law. Permission is hereby granted, free of +charge, to any person obtaining a copy of this software and associated +documentation (the “Software”), to use the Software without restriction, +including without limitation the rights to copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +others to do so, subject to the following conditions: + + 1. This LICENSE AND DISCLAIMER, including the copyright notice, shall be + included in all copies of the Software, including copies of substantial + portions of the Software; + + 2. JHU/APL assumes no obligation to provide support of any kind with regard + to the Software. This includes no obligation to provide assistance in using + the Software nor to provide updated versions of the Software; and + + 3. THE SOFTWARE AND ITS DOCUMENTATION ARE PROVIDED AS IS AND WITHOUT ANY + EXPRESS OR IMPLIED WARRANTIES WHATSOEVER. ALL WARRANTIES INCLUDING, BUT NOT + LIMITED TO, PERFORMANCE, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, + AND NONINFRINGEMENT ARE HEREBY DISCLAIMED. USERS ASSUME THE ENTIRE RISK AND + LIABILITY OF USING THE SOFTWARE. USERS ARE ADVISED TO TEST THE SOFTWARE + THOROUGHLY BEFORE RELYING ON IT. IN NO EVENT SHALL THE JOHNS HOPKINS + UNIVERSITY BE LIABLE FOR ANY DAMAGES WHATSOEVER, INCLUDING, WITHOUT + LIMITATION, ANY LOST PROFITS, LOST SAVINGS OR OTHER INCIDENTAL OR + CONSEQUENTIAL DAMAGES, ARISING OUT OF THE USE OR INABILITY TO USE THE + SOFTWARE.” diff --git a/doc/future.txt b/doc/future.txt new file mode 100644 index 0000000..d4afdbd --- /dev/null +++ b/doc/future.txt @@ -0,0 +1,62 @@ +Design concepts for "v2" rewrite of kvspool +------------------------------------------- +1. Support multi-writer, multi-reader from same spool +2. Use a memory-mapped file for reading/writing spool data so that: + (1) I/O occurs through shared memory even without a ramdisk, while + (2) data is still persisted back to disk +3. Support for multi-writers requires a synchronization mechanism. + (1) This is one of the functions of the "control file". + (a) this file exists alongside the spool data file + (b) by flock'ing it (or fcntl lock on a region of it), one writer + can gain exclusive write (which applies to the spool data file too); + a second level of record-locking using fcntl lock on the spool data + file can act as a redundant safeguard + (c) the control file has the min and max sequence number in it + (d) the max sequence number is just the "frame number" of the next + frame to be written + (e) the min sequence number is incremented (sometimes by an increment + greater than one) when the writer is overwriting previous frame(s). + It's purpose is explained under the "Support for multi-readers" later. + (f) The offset of the min and max frames are also stored + (g) The control block may also contain a few time-series on write rates. + (i) It would also be possible to place the control file into the data + spool itself, in which case its a "control block" of fixed size + at the beginning; this would eliminate some failure modes and + reduce the file descriptor bookkeeping by one +4. Spool data file is a single, large, circular data buffer + (a) It is pre-created prior to data being written to reserve the space + (b) This requires that it be a non-sparse file + (c) It is used as a cyclic buffer + (d) When the end is reached, a new frame may not quite fit at the end, + in which case the frame starts at the beginning of the file; but + this requires that the frame's content-length may differ from its + stored length (so that the frame that ends up at the end of the + buffer can be adjusted to consume the full remaining space). + (e) Thus the frame format is + (1) sequence number + (2) storage length + (3) content length + (4) data (in JSON) + (f) The single large data file replaces the kvspool-v1 approach + where ten sequenced files contain the spool data, and old files + are deleted as new files are written. The v1 logic requires + detection of new files in the spool, although its advantegous + in that read/write through standard (non-mmap) calls does not + swap in the entire data spool as the v2 approach may tend to do +5. Support for multi-readers + (1) since readers that are inactive for a long time may get to the point + that their next read position is potentially invalid (due to a + writer wraparound that puts a new frame into the read area), + (a) the reader that is entering the 'read' state will first + lock the control file, acquire the minimum sequence number + to see if its exceeded its own read position + (i) If it has, then the reader has experienced frame loss + and adjusts its next-read-position to the min frame + (ii) if not, the reader can then record-lock the spool + data and read the next frame + (iii) note that if the max sequence number is the same as + the read position, then the reader needs to block + (by placing inotify on the control file, unlocking + and going into a select/epoll). + (2) If reader needs persistence for its read position it should + store its own sequence number and identifier in the spool dir diff --git a/doc/kvspool-mini.png b/doc/kvspool-mini.png new file mode 100644 index 0000000000000000000000000000000000000000..f0855514581fc71dca5923395f75a525442cd467 GIT binary patch literal 1589 zcmV-52Fm$~P)7>tBOO*HZaMNkr+Rvrb!h+ug}-js)mQp)&Y=Xy7% zdv5OOHMGS$$;r7pJO7#4o!vRJ-3qWDs3K1dm5K_0Dpf75Z%^_RCK0J<64P;~h^!Kk z%_34MB2^-C^W}E%N+A&$FCwc&WSfZW7m;I0@WCJfJAmhaAAuWyLZEqWuKurS8Sn~l z3Fr=VNCFN9NmZ*=btf<@XZP}is-9KVmB75Z;Nx~uO8Kced6lO)N|feOlrczoY8aFc zQX`;(qyQ1=1PqL@&jP;#B@zBURo!T~e!%r!-$_;dCV}0)z$oBa;5T47FdCSzss}`* zH_#2}i2E>q0UQQ~0|V^$cdEKRgN+hk2v7>N0JZ>MtLokinC`$sKrh^)-vTUG)q~lN zEgPy(RpK5%?YYC+U4i$28si@a<^WT06OoeN0zBsF83AmxzQe%uOm-ta%)%{#*MO10 z65wys)yHJ!0l#|o=K!0473Rw&(r7Ho<1l$5$0gSiy zD&QP2IUwKLe7Klk>=lWFWW3J6QD6bk(&P64PMQ2O0bM1)1)z5VC$9slfkhGiNV8Ms z@rr@-#$O9`@cR1OQu(-F)Q#{U**=JA`GZ3QOOOOSA9rp-|Y zJo(p6zLs9#DPT#q4jN{x>021++W-uW@OGK}n1K8Y>pKHnZM;@MrSXRbFlE+v8t7uY zt8gh-fdK)`FzY)86kXOJ0c~;VU~~rg9;8!C-$>sNKF=noFy?7)GIyJ<3&3okorf;~ znj{EPG3hitA+n49O4*EHn&MKe0uv*^)9I3hnna;O&UqKH&S9=Lq~e zNsj{W;);cqxR3lw5&2R?S|u0*xIzb90R--w2X42-(HVPp$pMoSWt;BGfE#hui~?_F zz%&8&n%qoWV0YtvVC(YW(%=-_wN>px)qFIf9|Yah*Ew=#lx$=Z7|V2-E8 ze>v0LX;s|=OtoxmOdA~Ih@Za2NBRrLc^y$8250R|<| z)r|DYuCVtudl!pHyBN$h*4!S!l!!=+0HzJ@yyna1M0F{_KQi74plb}S6X~~rU_!9gSupW0dn}xd@I|$qtW9MPq*=MWCPqXnK z1qR1pM&hzWIqf$}yR5|LYR3k2YP;5OXEtW(v^0r_3PO~B*0K$YUoEt`Nfs(LPitwP*t z?}a-zZ8tlyS2iNj(J-aBYHvHRR#neLBGMIDp{ncZ&$Ha=iU|_#z v0.5, February 2012 -kvspool: a Linux-based C library used to read and write data streams +kvspool (a key-value spool):: + a Linux-based C library used to read and write data streams made of discrete frames; with network replication, snapshot/replay, bounded disk consumption and using key-value sets as the streaming unit. kvspool's niche --------------- -Kvspool falls somewhere between the Unix pipe, a record-oriented database and a -message-passing library. It's not any of those, and yet it resembles them all. It's simple -to use, but reflects a particular set of design goals. +Kvspool falls somewhere between the Unix pipe, a file-backed queue and a message-passing +library. It's not quite any of them, yet resembles them all. It reflects a personal set +of design goals. -* like the Unix pipe, it streams data from one program to another -* its data capacity is configurable -* the data is disk or ramdisk-resident -* the data stream is composed of distinct messages (or "frames") -* each "frame" is a set of key-value pairs (aka a "dictionary", "hash", etc) -* the data stream can be copied off to a "snapshot" at any time -* the data stream or snapshot supports rewind and replay -* streams can be sent over a network +* the spool is used to "stream" (transmit) data frames from one program to another +* the spool frames are each a "hash"- a set of key-value pairs (aka a dictionary) +* the spool writer never blocks, even if the reader is slow, absent, or crashes +* the spool is a disk- or ramdisk-resident buffer of a configurable size +* the spool reader gets frames from the writer via the file system only +* the spool reader can exit, restart, and "catch up" with the writer +* the spool reader blocks (waiting for new data) when its caught up +* the spool reader loses data if its absent/offline/slow enough +* the spool frames remain on disk til their space is reclaimed +* the spool can be copied off to a "snapshot" at any time +* the spool supports rewind and replay +* the spool can be sent over a network + +Sneak peak +~~~~~~~~~~ +Here's an example of writing from Perl and reading from C. + +[options="header"] +|=============================================================================== +| Perl writer | C reader +| use KVSpool; | #include "kvspool.h" +| my $v = KVSpool->new("spool"); | void *sp = kv_spoolreader_new("spool"); +| my $h = {'day'=>'Wed','temp'=>37}; | void *set = kv_set_new(); +| $v->write($h); | kv_spool_read(sp,set,1); +|=============================================================================== + +Rewind and replay +~~~~~~~~~~~~~~~~~ +Kvspool keeps the data frames, even after they've been read-- til space needs to be +reclaimed. (So the spool is a like a long reel of tape spliced together at the ends). +There are several nice outcomes of this: + +* You have a history, or a "rear-view window" of the stream from writer to reader +* Because you have this history of the stream, you can copy it off +* You can take it back to a development or test environment +* You can "rewind" and "replay" the spool for testing + +Canned data +^^^^^^^^^^^ +For developers, kvspool can be a convenient way to take "canned" data from a production +environment. Just copy the spool. Now the data is canned. The developer can now take it +on a laptop (where the writer is not even necessary), rewind it, and use it as input. Platform ~~~~~~~~ Kvspool is written for Linux, and has support for C, Perl, Python and Java. -The C library does not depend on any other libraries; if you have ZeroMQ (2.x -or 3.x) installed and the Jansson library installed, additional utilities for network -replication of spools are built. +While the C library does not depend on any other libraries, it's recommended to have if +you have *ZeroMQ* (2.x or 3.x) and the *Jansson* library installed, additional utilities for +network replication of spools are built. License ~~~~~~~ -See the LICENSE file. Kvspool is free and open source. +See the link:LICENSE[LICENSE] file. Kvspool is free and open source. -Motivation -~~~~~~~~~~ -It all started with a sensor. Like any sensor this one produced an endless series of +Resources & Help +~~~~~~~~~~~~~~~~ +News about software updates are posted to the author's blog: http://tkhanson.net/blog. +Contact the author directly at tdh@tkhanson.net if you have questions or other issues. + +History & Motivation +~~~~~~~~~~~~~~~~~~~~ +It started with a sensor. Like any sensor this one produced an endless series of measurements. The measurements were fed into another process. How? With a Unix pipe: sensor | analyzer Beatiful and concise, but: -* what happens if 'sensor' produces data faster than 'analyzer' can read it? -* What happens to 'sensor' if 'analyzer' crashes? +* what happens if `sensor` produces data faster than `analyzer` can read it? +* What happens to `sensor` if `analyzer` crashes? -If 'sensor' is doing something important- the pipe is not robust because 'sensor' gets -'blocked' (put to sleep) if 'analyzer' reads the pipe too slowly-- and worse yet, -any bugs that crash 'analyzer' thereby break the pipe and crash 'sensor' too. +If `sensor` is doing something important- the pipe is not robust because `sensor` gets +blocked (put to sleep) if 'analyzer' reads the pipe too slowly-- and worse yet, +any bugs that crash '`nalyzer` thereby break the pipe and crash `sensor` too. In search of ^^^^^^^^^^^^ -The `sensor | analyzer` pipeline could be replaced many ways: for example 'sensor' could -write to a database, which 'analyzer' could poll periodically. But Unix people dislike +The `sensor | analyzer` pipeline could be replaced many ways: for example `sensor` could +write to a database, which `analyzer` could poll periodically. But Unix people dislike polling. It says "I couldn't figure out an event-driven solution to this problem". We could use shared memory, and semaphores, etc. However, there's also a Unix mindset that says "everything is a file"-- so why shouldn't our data stream be one too? In other @@ -61,18 +101,18 @@ benefits that confers (for example, the ability to copy it easily) and yet still event-driven model where the reader is woken up only when new data is available? (Yes, we can, using inotify). The wish list became, - #. stream should be a file (can be on a ram disk) - #. reader should be event driven - #. let the user configure how much disk space to allocate to the stream - #. drop old data (whether its read or unread) when the stream fills up - #. put framing into the stream so that we can read and write whole messages - #. use key-value sets (aka a dictionary or hash) as the data unit - #. copy a "live" data stream to a frozen "snapshot" - #. support rewind and replay. - #. work locally or over a network. - #. insulate writer from reader (so much that either can be absent or sporadically present) - #. work with many languages. - #. easy to use. +* stream should be a file (can be on a ram disk) +* reader should be event driven +* let the user configure how much disk space to allocate to the stream +* drop old data (whether its read or unread) when the stream fills up +* put framing into the stream so that we can read and write whole messages +* use key-value sets (aka a dictionary or hash) as the data unit +* copy a "live" data stream to a frozen "snapshot" +* support rewind and replay. +* work locally or over a network. +* insulate writer from reader (so much that either can be absent or sporadically present) +* work with many languages. +* easy to use. Kvspool does these things. It's not a sophisticated suite. It's just a tool in the Unix tradition that does one thing and tries to do it well. @@ -85,12 +125,6 @@ or rewind a stream, to watch its status, set up network replication, and so on. used to read and write the stream is extremely simple: key-value sets (dictionary or hash are common names for this data structure) are simply read from the stream or written to it. -The best feature -^^^^^^^^^^^^^^^^ -Perhaps more than any other feature, the ability to copy a "live" data stream to an -offline "snapshot", and then replay it as often as necessary, for testing or algorithm -development, can be very valuable. Testing with a consistent data set becomes easy. - Does kvspool keep data after its been read? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Yes, for two reasons. Kvspool keeps data, even after its been read, up to the maximum @@ -116,11 +150,6 @@ To build it: This builds and installs the C library and utilities, and if the prerequisite packages are installed, it builds the Perl, Python and Java bindings, and ZeroMQ-based utilities. -Resources & Help -~~~~~~~~~~~~~~~~ -News about software updates are posted to the author's blog: http://tkhanson.net/blog. -Contact the author directly at tdh@tkhanson.net if you have questions or other issues. - Basics ------ @@ -146,17 +175,20 @@ Write data to spool We show a simple example of using the spool in Perl, Python and C here. .Perl +[source,perl] use KVSpool; my $h = {'day' => 'Wednesday', 'user' => 'Troy'}; my $v = KVSpool->new("spool"); $v->write($h); .Python +[source,python] import kvpy d = {"day":"Wednesday","user":"Troy"} kvpy.kvpy_write("spool",d) .C +[source,c] #include "kvspool.h" ... void *sp = kv_spoolwriter_new("spool"); @@ -170,15 +202,18 @@ We show a simple example of using the spool in Perl, Python and C here. Read data from spool ~~~~~~~~~~~~~~~~~~~~ .Perl +[source,perl] use KVSpool; my $v = KVSpool->new("spool"); my $h = $v->read(); .Python +[source,python] import kvpy d = kvpy.kvpy_read("spool") .C +[source,c] #include "kvspool.h" ... void *sp = kv_spoolreader_new("spool"); @@ -228,7 +263,7 @@ is easy to overcome when multiple independent readers each need their own copy o Persistent read position ^^^^^^^^^^^^^^^^^^^^^^^^ The spool records the reader position internally. If a reader exits, then restarts, it -picks up where it left off. (The `kvsp-reset` utility can be used to reset the reader +picks up where it left off. (The `kvsp-rewind` utility can be used to reset the reader position to the beginning, for replay purposes). Because the read position is stored in the spool, you can see it using `kvsp-status`. @@ -257,12 +292,14 @@ Programs written against the kvspool API can be linked with -lkvspool. Reader API ^^^^^^^^^^ +[source,c] void *kv_spoolreader_new(const char *dir); int kv_spool_read(void*sp, void *set, int blocking); void kv_spoolreader_free(void*); Writer API ^^^^^^^^^^ +[source,c] void *kv_spoolwriter_new(const char *dir); int kv_spool_write(void*sp, void *set); void kv_spoolwriter_free(void*); @@ -271,6 +308,7 @@ Dictionary API ^^^^^^^^^^^^^^ The `void *set` in the C API is a dictionary data structure in C. +[source,c] void* kv_set_new(void); void kv_set_free(void*); void kv_set_clear(void*); @@ -291,6 +329,7 @@ The `void *set` in the C API is a dictionary data structure in C. A C program can iterate through all the keys/values like: +[source,c] kv_t *kv = NULL; while ( (kv = kv_next(set, kv))) { printf("key is %s\n", kv->key); @@ -299,23 +338,27 @@ A C program can iterate through all the keys/values like: Reset API ~~~~~~~~~ -This is the programmatic equal of the `kvsp-reset` command: +This is the programmatic equal of the `kvsp-rewind` command: +[source,c] void sp_reset(const char *dir); Perl ~~~~ In Perl this is how to use the module and open a spool for reading or writing: +[source,perl] use KVSpool; my $v = KVSpool->new("spool"); Then to read: +[source,perl] my $h = $v->read(); # returns a hash reference Similarly to write: +[source,perl] $v->write($h); # where h is a hash reference Python @@ -323,6 +366,7 @@ Python As of the current version kvspool only has a procedural interface for Python. If d is a dicionary then the API to write or read a frame is simply: +[source,python] import kvpy kvpy.kvpy_write("spool",d) d = kvpy.kvpy_read("spool") @@ -339,7 +383,7 @@ Basic |command | example |kvsp-size | kvsp-size -s 1G spool |kvsp-status | kvsp-status spool -|kvsp-reset | kvsp-reset spool +|kvsp-rewind | kvsp-rewind spool |kvsp-tee | kvsp-tee -s spool-in spool-copy1 spool-copy2 |=============================================================================== @@ -349,7 +393,7 @@ run later, after the spool already exists and has data, it is resized. Run `kvsp-status` to see what percentage of the spool has been consumed by a reader. -The `kvsp-reset` command resets the reader position to the beginning (oldest frame) in the +The `kvsp-rewind` command resets the reader position to the beginning (oldest frame) in the spool. Use this command in order to "replay" the spooled data. Disconnect (terminate) any readers before running this command. @@ -453,7 +497,7 @@ To snapshot a spool, just copy it: copying the spool). With the snapshot copied off, it can now be "replayed" as often as needed to develop new versions of the software that reads it. - kvsp-reset snapshot + kvsp-rewind snapshot Since the snapshot is "canned" real data, but not being written to any longer, it is useful as a consistent data set to test new versions of software. The other major benefit @@ -522,7 +566,7 @@ Rough edges: More sweeping ideas for a possible future "v2" rewrite: -* Support multi-writer, multi-reader (see multi_future.txt) +* Support multi-writer, multi-reader (see future.txt) * Replace segemented data files with one memory mapped, circular file * Use JSON internally diff --git a/kvspool.h b/include/kvspool.h similarity index 98% rename from kvspool.h rename to include/kvspool.h index c1ff05f..54d42ae 100644 --- a/kvspool.h +++ b/include/kvspool.h @@ -51,7 +51,7 @@ void sp_attrition(char *dir); *****************************************************************************/ typedef struct { int pct_consumed; } kv_stat_t; int kv_stat(const char *dir, kv_stat_t *stats); -void sp_reset(const char *dir); +void sp_rewind(const char *dir); typedef struct { size_t dir_max; diff --git a/kvspool_internal.h b/include/kvspool_internal.h similarity index 100% rename from kvspool_internal.h rename to include/kvspool_internal.h diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..44010fc --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,5 @@ +AM_CFLAGS = -fPIC -I../include +lib_LIBRARIES = libkvspool.a +libkvspool_a_SOURCES = kvspool.c kvspoolw.c kvspoolr.c kvspool_attrition.c tpl.c +include_HEADERS = ../include/kvspool.h + diff --git a/kvspool.c b/src/kvspool.c similarity index 100% rename from kvspool.c rename to src/kvspool.c diff --git a/kvspool_attrition.c b/src/kvspool_attrition.c similarity index 100% rename from kvspool_attrition.c rename to src/kvspool_attrition.c diff --git a/kvspoolr.c b/src/kvspoolr.c similarity index 99% rename from kvspoolr.c rename to src/kvspoolr.c index 24a488d..f44d31f 100644 --- a/kvspoolr.c +++ b/src/kvspoolr.c @@ -331,7 +331,7 @@ void kv_spoolreader_free(void *_sp) { free(sp); } -void sp_reset(const char *dir) { +void sp_rewind(const char *dir) { char *path, **p; int sr_fd,rp; diff --git a/kvspoolw.c b/src/kvspoolw.c similarity index 100% rename from kvspoolw.c rename to src/kvspoolw.c diff --git a/tpl.c b/src/tpl.c similarity index 100% rename from tpl.c rename to src/tpl.c diff --git a/utils/.gitignore b/utils/.gitignore index 6e48c33..8945f8b 100644 --- a/utils/.gitignore +++ b/utils/.gitignore @@ -5,7 +5,7 @@ kvsp-pub kvsp-speed kvsp-status kvsp-mod -kvsp-reset +kvsp-rewind kvsp-spr kvsp-spw *.o diff --git a/utils/Makefile.am b/utils/Makefile.am index 1be6ce3..57f3b2f 100644 --- a/utils/Makefile.am +++ b/utils/Makefile.am @@ -1,7 +1,7 @@ AM_CFLAGS = -I.. -I../include -LIBSPOOL = -L.. -lkvspool +LIBSPOOL = -L../src -lkvspool bin_PROGRAMS = kvsp-spr kvsp-spw kvsp-tee kvsp-size kvsp-status \ - kvsp-speed kvsp-mod kvsp-reset \ + kvsp-speed kvsp-mod kvsp-rewind \ ramdisk kvsp_spr_LDADD = $(LIBSPOOL) @@ -11,7 +11,7 @@ kvsp_size_LDADD = $(LIBSPOOL) kvsp_status_LDADD = $(LIBSPOOL) kvsp_speed_LDADD = $(LIBSPOOL) kvsp_mod_LDADD = $(LIBSPOOL) -kvsp_reset_LDADD = $(LIBSPOOL) +kvsp_rewind_LDADD = $(LIBSPOOL) kvsp_pub_LDADD = $(LIBSPOOL) kvsp_sub_LDADD = $(LIBSPOOL) @@ -24,13 +24,13 @@ endif endif # to get a rebuild of the utilities when ../libkvspool.a changes: -kvsp_spr_DEPENDENCIES = ../libkvspool.a -kvsp_spw_DEPENDENCIES = ../libkvspool.a -kvsp_tee_DEPENDENCIES = ../libkvspool.a -kvsp_size_DEPENDENCIES = ../libkvspool.a -kvsp_status_DEPENDENCIES = ../libkvspool.a -kvsp_speed_DEPENDENCIES = ../libkvspool.a -kvsp_mod_DEPENDENCIES = ../libkvspool.a -kvsp_reset_DEPENDENCIES = ../libkvspool.a -kvsp_sub_DEPENDENCIES = ../libkvspool.a -kvsp_pub_DEPENDENCIES = ../libkvspool.a +kvsp_spr_DEPENDENCIES = ../src/libkvspool.a +kvsp_spw_DEPENDENCIES = ../src/libkvspool.a +kvsp_tee_DEPENDENCIES = ../src/libkvspool.a +kvsp_size_DEPENDENCIES = ../src/libkvspool.a +kvsp_status_DEPENDENCIES = ../src/libkvspool.a +kvsp_speed_DEPENDENCIES = ../src/libkvspool.a +kvsp_mod_DEPENDENCIES = ../src/libkvspool.a +kvsp_rewind_DEPENDENCIES = ../src/libkvspool.a +kvsp_sub_DEPENDENCIES = ../src/libkvspool.a +kvsp_pub_DEPENDENCIES = ../src/libkvspool.a diff --git a/utils/kvsp-reset.c b/utils/kvsp-rewind.c similarity index 96% rename from utils/kvsp-reset.c rename to utils/kvsp-rewind.c index 02d3f6e..573e41f 100644 --- a/utils/kvsp-reset.c +++ b/utils/kvsp-rewind.c @@ -24,7 +24,7 @@ int main(int argc, char *argv[]) { if (optind < argc) dir=argv[optind++]; else usage(exe); - sp_reset(dir); + sp_rewind(dir); return 0; }