AtHeartEngineer/kvspool
1
0
Fork 0
mirror of https://github.com/JHUAPL/kvspool.git synced 2026-01-07 22:13:49 -05:00
Code Issues Packages Projects Releases Wiki Activity

work in progress

Browse Source
This commit is contained in:
Troy D. Hanson
2012-02-26 19:46:41 -05:00
parent ee9b3cd9de
commit 8a100f8626
20 changed files with 227 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Makefile.am
View File

@@ -1,9 +1,4 @@
AM_CFLAGS = -fPIC -I./include
lib_LIBRARIES = libkvspool.a
libkvspool_a_SOURCES = kvspool.c kvspoolw.c kvspoolr.c kvspool_attrition.c tpl.c
include_HEADERS = kvspool.h
SUBDIRS = . utils
SUBDIRS = src utils
if HAVE_PYTHON
SUBDIRS += kvpy

8
README
View File

@@ -1,11 +1,7 @@
kvspool data streaming utilities
by Troy D. Hanson
Documentation for kvspool is at:
http://troydhanson.github.com/kvspool
Special thanks to:
Trevor Adams
JT Halbert
Jeff James

10
TODO
View File

@@ -1,10 +0,0 @@
4. Add documentation/slides
6. Change Python to have object wrapper
#done
2. Add missing KVJava
3. Move sysutils to their own repo
5. Remove 'base' parameter everywhere
7. zcon and zcontrol should be in a ZeroMQ add-on's repo
8. kvsp-import/export should be subsumed by kvsp-pub/sub
9. put libts in external snippets repo

4
configure.ac
View File

@@ -1,7 +1,7 @@
AC_PREREQ(2.59)
AC_INIT([kvspool], [1.0], [tdh@tkhanson.net])
AC_CONFIG_SRCDIR(kvspool.c)
AC_CONFIG_SRCDIR(src/kvspool.c)
AC_CONFIG_AUX_DIR(config)
AC_CONFIG_HEADERS(config/config.h)
AM_INIT_AUTOMAKE
@@ -24,6 +24,6 @@ AM_CONDITIONAL(HAVE_PYTHON,test "x$pythonexists" = "xyes")
AC_CHECK_PROG(PERL,perl,perl)
AM_CONDITIONAL(HAVE_PERL,test "x$PERL" != "x")
AC_CONFIG_FILES(Makefile utils/Makefile)
AC_CONFIG_FILES(Makefile src/Makefile utils/Makefile)
AC_OUTPUT

8
doc/CREDITS Normal file
View File

@@ -0,0 +1,8 @@
kvspool was developed in 2011 by Troy D. Hanson
Special thanks to:
JHU/APL OTT
Trevor Adams
JT Halbert
Jeff James
Nick Clote

31
doc/LICENSE Normal file
View File

@@ -0,0 +1,31 @@
LICENSE AND DISCLAIMER
Copyright (c) 2011 The Johns Hopkins University/Applied Physics Laboratory
This software was developed at The Johns Hopkins University/Applied Physics
Laboratory (“JHU/APL”) that is the author thereof under the “work made for
hire” provisions of the copyright law. Permission is hereby granted, free of
charge, to any person obtaining a copy of this software and associated
documentation (the “Software”), to use the Software without restriction,
including without limitation the rights to copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to permit
others to do so, subject to the following conditions:
1. This LICENSE AND DISCLAIMER, including the copyright notice, shall be
included in all copies of the Software, including copies of substantial
portions of the Software;
2. JHU/APL assumes no obligation to provide support of any kind with regard
to the Software. This includes no obligation to provide assistance in using
the Software nor to provide updated versions of the Software; and
3. THE SOFTWARE AND ITS DOCUMENTATION ARE PROVIDED AS IS AND WITHOUT ANY
EXPRESS OR IMPLIED WARRANTIES WHATSOEVER. ALL WARRANTIES INCLUDING, BUT NOT
LIMITED TO, PERFORMANCE, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
AND NONINFRINGEMENT ARE HEREBY DISCLAIMED. USERS ASSUME THE ENTIRE RISK AND
LIABILITY OF USING THE SOFTWARE. USERS ARE ADVISED TO TEST THE SOFTWARE
THOROUGHLY BEFORE RELYING ON IT. IN NO EVENT SHALL THE JOHNS HOPKINS
UNIVERSITY BE LIABLE FOR ANY DAMAGES WHATSOEVER, INCLUDING, WITHOUT
LIMITATION, ANY LOST PROFITS, LOST SAVINGS OR OTHER INCIDENTAL OR
CONSEQUENTIAL DAMAGES, ARISING OUT OF THE USE OR INABILITY TO USE THE
SOFTWARE.”

62
doc/future.txt Normal file
View File

@@ -0,0 +1,62 @@
Design concepts for "v2" rewrite of kvspool
-------------------------------------------
1. Support multi-writer, multi-reader from same spool
2. Use a memory-mapped file for reading/writing spool data so that:
(1) I/O occurs through shared memory even without a ramdisk, while
(2) data is still persisted back to disk
3. Support for multi-writers requires a synchronization mechanism.
(1) This is one of the functions of the "control file".
(a) this file exists alongside the spool data file
(b) by flock'ing it (or fcntl lock on a region of it), one writer
can gain exclusive write (which applies to the spool data file too);
a second level of record-locking using fcntl lock on the spool data
file can act as a redundant safeguard
(c) the control file has the min and max sequence number in it
(d) the max sequence number is just the "frame number" of the next
frame to be written
(e) the min sequence number is incremented (sometimes by an increment
greater than one) when the writer is overwriting previous frame(s).
It's purpose is explained under the "Support for multi-readers" later.
(f) The offset of the min and max frames are also stored
(g) The control block may also contain a few time-series on write rates.
(i) It would also be possible to place the control file into the data
spool itself, in which case its a "control block" of fixed size
at the beginning; this would eliminate some failure modes and
reduce the file descriptor bookkeeping by one
4. Spool data file is a single, large, circular data buffer
(a) It is pre-created prior to data being written to reserve the space
(b) This requires that it be a non-sparse file
(c) It is used as a cyclic buffer
(d) When the end is reached, a new frame may not quite fit at the end,
in which case the frame starts at the beginning of the file; but
this requires that the frame's content-length may differ from its
stored length (so that the frame that ends up at the end of the
buffer can be adjusted to consume the full remaining space).
(e) Thus the frame format is
(1) sequence number
(2) storage length
(3) content length
(4) data (in JSON)
(f) The single large data file replaces the kvspool-v1 approach
where ten sequenced files contain the spool data, and old files
are deleted as new files are written. The v1 logic requires
detection of new files in the spool, although its advantegous
in that read/write through standard (non-mmap) calls does not
swap in the entire data spool as the v2 approach may tend to do
5. Support for multi-readers
(1) since readers that are inactive for a long time may get to the point
that their next read position is potentially invalid (due to a
writer wraparound that puts a new frame into the read area),
(a) the reader that is entering the 'read' state will first
lock the control file, acquire the minimum sequence number
to see if its exceeded its own read position
(i) If it has, then the reader has experienced frame loss
and adjusts its next-read-position to the min frame
(ii) if not, the reader can then record-lock the spool
data and read the next frame
(iii) note that if the max sequence number is the same as
the read position, then the reader needs to block
(by placing inotify on the control file, unlocking
and going into a select/epoll).
(2) If reader needs persistence for its read position it should
store its own sequence number and identifier in the spool dir

BIN
doc/kvspool-mini.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.6 KiB

154
kvspool.txt → doc/kvspool.txt
View File

@@ -3,56 +3,96 @@ kvspool: a tool for data streams
Troy D. Hanson <tdh@tkhanson.net>
v0.5, February 2012
kvspool: a Linux-based C library used to read and write data streams
kvspool (a key-value spool)::
a Linux-based C library used to read and write data streams
made of discrete frames; with network replication, snapshot/replay,
bounded disk consumption and using key-value sets as the streaming unit.
kvspool's niche
---------------
Kvspool falls somewhere between the Unix pipe, a record-oriented database and a
message-passing library. It's not any of those, and yet it resembles them all. It's simple
to use, but reflects a particular set of design goals.
Kvspool falls somewhere between the Unix pipe, a file-backed queue and a message-passing
library. It's not quite any of them, yet resembles them all. It reflects a personal set
of design goals.
* like the Unix pipe, it streams data from one program to another
* its data capacity is configurable
* the data is disk or ramdisk-resident
* the data stream is composed of distinct messages (or "frames")
* each "frame" is a set of key-value pairs (aka a "dictionary", "hash", etc)
* the data stream can be copied off to a "snapshot" at any time
* the data stream or snapshot supports rewind and replay
* streams can be sent over a network
* the spool is used to "stream" (transmit) data frames from one program to another
* the spool frames are each a "hash"- a set of key-value pairs (aka a dictionary)
* the spool writer never blocks, even if the reader is slow, absent, or crashes
* the spool is a disk- or ramdisk-resident buffer of a configurable size
* the spool reader gets frames from the writer via the file system only
* the spool reader can exit, restart, and "catch up" with the writer
* the spool reader blocks (waiting for new data) when its caught up
* the spool reader loses data if its absent/offline/slow enough
* the spool frames remain on disk til their space is reclaimed
* the spool can be copied off to a "snapshot" at any time
* the spool supports rewind and replay
* the spool can be sent over a network
Sneak peak
~~~~~~~~~~
Here's an example of writing from Perl and reading from C.
[options="header"]
|===============================================================================
| Perl writer | C reader
| use KVSpool; | #include "kvspool.h"
| my $v = KVSpool->new("spool"); | void *sp = kv_spoolreader_new("spool");
| my $h = {'day'=>'Wed','temp'=>37}; | void *set = kv_set_new();
| $v->write($h); | kv_spool_read(sp,set,1);
|===============================================================================
Rewind and replay
~~~~~~~~~~~~~~~~~
Kvspool keeps the data frames, even after they've been read-- til space needs to be
reclaimed. (So the spool is a like a long reel of tape spliced together at the ends).
There are several nice outcomes of this:
* You have a history, or a "rear-view window" of the stream from writer to reader
* Because you have this history of the stream, you can copy it off
* You can take it back to a development or test environment
* You can "rewind" and "replay" the spool for testing
Canned data
^^^^^^^^^^^
For developers, kvspool can be a convenient way to take "canned" data from a production
environment. Just copy the spool. Now the data is canned. The developer can now take it
on a laptop (where the writer is not even necessary), rewind it, and use it as input.
Platform
~~~~~~~~
Kvspool is written for Linux, and has support for C, Perl, Python and Java.
The C library does not depend on any other libraries; if you have ZeroMQ (2.x
or 3.x) installed and the Jansson library installed, additional utilities for network
replication of spools are built.
While the C library does not depend on any other libraries, it's recommended to have if
you have *ZeroMQ* (2.x or 3.x) and the *Jansson* library installed, additional utilities for
network replication of spools are built.
License
~~~~~~~
See the LICENSE file. Kvspool is free and open source.
See the link:LICENSE[LICENSE] file. Kvspool is free and open source.
Motivation
~~~~~~~~~~
It all started with a sensor. Like any sensor this one produced an endless series of
Resources & Help
~~~~~~~~~~~~~~~~
News about software updates are posted to the author's blog: http://tkhanson.net/blog.
Contact the author directly at tdh@tkhanson.net if you have questions or other issues.
History & Motivation
~~~~~~~~~~~~~~~~~~~~
It started with a sensor. Like any sensor this one produced an endless series of
measurements. The measurements were fed into another process. How? With a Unix pipe:
sensor | analyzer
Beatiful and concise, but:
* what happens if 'sensor' produces data faster than 'analyzer' can read it?
* What happens to 'sensor' if 'analyzer' crashes?
* what happens if `sensor` produces data faster than `analyzer` can read it?
* What happens to `sensor` if `analyzer` crashes?
If 'sensor' is doing something important- the pipe is not robust because 'sensor' gets
'blocked' (put to sleep) if 'analyzer' reads the pipe too slowly-- and worse yet,
any bugs that crash 'analyzer' thereby break the pipe and crash 'sensor' too.
If `sensor` is doing something important- the pipe is not robust because `sensor` gets
blocked (put to sleep) if 'analyzer' reads the pipe too slowly-- and worse yet,
any bugs that crash '`nalyzer` thereby break the pipe and crash `sensor` too.
In search of
^^^^^^^^^^^^
The `sensor | analyzer` pipeline could be replaced many ways: for example 'sensor' could
write to a database, which 'analyzer' could poll periodically. But Unix people dislike
The `sensor | analyzer` pipeline could be replaced many ways: for example `sensor` could
write to a database, which `analyzer` could poll periodically. But Unix people dislike
polling. It says "I couldn't figure out an event-driven solution to this problem". We
could use shared memory, and semaphores, etc. However, there's also a Unix mindset that
says "everything is a file"-- so why shouldn't our data stream be one too? In other
@@ -61,18 +101,18 @@ benefits that confers (for example, the ability to copy it easily) and yet still
event-driven model where the reader is woken up only when new data is available? (Yes, we
can, using inotify). The wish list became,
#. stream should be a file (can be on a ram disk)
#. reader should be event driven
#. let the user configure how much disk space to allocate to the stream
#. drop old data (whether its read or unread) when the stream fills up
#. put framing into the stream so that we can read and write whole messages
#. use key-value sets (aka a dictionary or hash) as the data unit
#. copy a "live" data stream to a frozen "snapshot"
#. support rewind and replay.
#. work locally or over a network.
#. insulate writer from reader (so much that either can be absent or sporadically present)
#. work with many languages.
#. easy to use.
* stream should be a file (can be on a ram disk)
* reader should be event driven
* let the user configure how much disk space to allocate to the stream
* drop old data (whether its read or unread) when the stream fills up
* put framing into the stream so that we can read and write whole messages
* use key-value sets (aka a dictionary or hash) as the data unit
* copy a "live" data stream to a frozen "snapshot"
* support rewind and replay.
* work locally or over a network.
* insulate writer from reader (so much that either can be absent or sporadically present)
* work with many languages.
* easy to use.
Kvspool does these things. It's not a sophisticated suite. It's just a tool in the
Unix tradition that does one thing and tries to do it well.
@@ -85,12 +125,6 @@ or rewind a stream, to watch its status, set up network replication, and so on.
used to read and write the stream is extremely simple: key-value sets (dictionary or hash
are common names for this data structure) are simply read from the stream or written to it.
The best feature
^^^^^^^^^^^^^^^^
Perhaps more than any other feature, the ability to copy a "live" data stream to an
offline "snapshot", and then replay it as often as necessary, for testing or algorithm
development, can be very valuable. Testing with a consistent data set becomes easy.
Does kvspool keep data after its been read?
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Yes, for two reasons. Kvspool keeps data, even after its been read, up to the maximum
@@ -116,11 +150,6 @@ To build it:
This builds and installs the C library and utilities, and if the prerequisite packages
are installed, it builds the Perl, Python and Java bindings, and ZeroMQ-based utilities.
Resources & Help
~~~~~~~~~~~~~~~~
News about software updates are posted to the author's blog: http://tkhanson.net/blog.
Contact the author directly at tdh@tkhanson.net if you have questions or other issues.
Basics
------
@@ -146,17 +175,20 @@ Write data to spool
We show a simple example of using the spool in Perl, Python and C here.
.Perl
[source,perl]
use KVSpool;
my $h = {'day' => 'Wednesday', 'user' => 'Troy'};
my $v = KVSpool->new("spool");
$v->write($h);
.Python
[source,python]
import kvpy
d = {"day":"Wednesday","user":"Troy"}
kvpy.kvpy_write("spool",d)
.C
[source,c]
#include "kvspool.h"
...
void *sp = kv_spoolwriter_new("spool");
@@ -170,15 +202,18 @@ We show a simple example of using the spool in Perl, Python and C here.
Read data from spool
~~~~~~~~~~~~~~~~~~~~
.Perl
[source,perl]
use KVSpool;
my $v = KVSpool->new("spool");
my $h = $v->read();
.Python
[source,python]
import kvpy
d = kvpy.kvpy_read("spool")
.C
[source,c]
#include "kvspool.h"
...
void *sp = kv_spoolreader_new("spool");
@@ -228,7 +263,7 @@ is easy to overcome when multiple independent readers each need their own copy o
Persistent read position
^^^^^^^^^^^^^^^^^^^^^^^^
The spool records the reader position internally. If a reader exits, then restarts, it
picks up where it left off. (The `kvsp-reset` utility can be used to reset the reader
picks up where it left off. (The `kvsp-rewind` utility can be used to reset the reader
position to the beginning, for replay purposes).
Because the read position is stored in the spool, you can see it using `kvsp-status`.
@@ -257,12 +292,14 @@ Programs written against the kvspool API can be linked with -lkvspool.
Reader API
^^^^^^^^^^
[source,c]
void *kv_spoolreader_new(const char *dir);
int kv_spool_read(void*sp, void *set, int blocking);
void kv_spoolreader_free(void*);
Writer API
^^^^^^^^^^
[source,c]
void *kv_spoolwriter_new(const char *dir);
int kv_spool_write(void*sp, void *set);
void kv_spoolwriter_free(void*);
@@ -271,6 +308,7 @@ Dictionary API
^^^^^^^^^^^^^^
The `void *set` in the C API is a dictionary data structure in C.
[source,c]
void* kv_set_new(void);
void kv_set_free(void*);
void kv_set_clear(void*);
@@ -291,6 +329,7 @@ The `void *set` in the C API is a dictionary data structure in C.
A C program can iterate through all the keys/values like:
[source,c]
kv_t *kv = NULL;
while ( (kv = kv_next(set, kv))) {
printf("key is %s\n", kv->key);
@@ -299,23 +338,27 @@ A C program can iterate through all the keys/values like:
Reset API
~~~~~~~~~
This is the programmatic equal of the `kvsp-reset` command:
This is the programmatic equal of the `kvsp-rewind` command:
[source,c]
void sp_reset(const char *dir);
Perl
~~~~
In Perl this is how to use the module and open a spool for reading or writing:
[source,perl]
use KVSpool;
my $v = KVSpool->new("spool");
Then to read:
[source,perl]
my $h = $v->read(); # returns a hash reference
Similarly to write:
[source,perl]
$v->write($h); # where h is a hash reference
Python
@@ -323,6 +366,7 @@ Python
As of the current version kvspool only has a procedural interface for Python. If d is a
dicionary then the API to write or read a frame is simply:
[source,python]
import kvpy
kvpy.kvpy_write("spool",d)
d = kvpy.kvpy_read("spool")
@@ -339,7 +383,7 @@ Basic
|command | example
|kvsp-size | kvsp-size -s 1G spool
|kvsp-status | kvsp-status spool
|kvsp-reset | kvsp-reset spool
|kvsp-rewind | kvsp-rewind spool
|kvsp-tee | kvsp-tee -s spool-in spool-copy1 spool-copy2
|===============================================================================
@@ -349,7 +393,7 @@ run later, after the spool already exists and has data, it is resized.
Run `kvsp-status` to see what percentage of the spool has been consumed by a reader.
The `kvsp-reset` command resets the reader position to the beginning (oldest frame) in the
The `kvsp-rewind` command resets the reader position to the beginning (oldest frame) in the
spool. Use this command in order to "replay" the spooled data. Disconnect (terminate) any
readers before running this command.
@@ -453,7 +497,7 @@ To snapshot a spool, just copy it:
copying the spool). With the snapshot copied off, it can now be "replayed" as often as
needed to develop new versions of the software that reads it.
kvsp-reset snapshot
kvsp-rewind snapshot
Since the snapshot is "canned" real data, but not being written to any longer, it is
useful as a consistent data set to test new versions of software. The other major benefit
@@ -522,7 +566,7 @@ Rough edges:
More sweeping ideas for a possible future "v2" rewrite:
* Support multi-writer, multi-reader (see multi_future.txt)
* Support multi-writer, multi-reader (see future.txt)
* Replace segemented data files with one memory mapped, circular file
* Use JSON internally

2
kvspool.h → include/kvspool.h
View File

@@ -51,7 +51,7 @@ void sp_attrition(char *dir);
*****************************************************************************/
typedef struct { int pct_consumed; } kv_stat_t;
int kv_stat(const char *dir, kv_stat_t *stats);
void sp_reset(const char *dir);
void sp_rewind(const char *dir);
typedef struct {
size_t dir_max;

0
kvspool_internal.h → include/kvspool_internal.h
View File

5
src/Makefile.am Normal file
View File

@@ -0,0 +1,5 @@
AM_CFLAGS = -fPIC -I../include
lib_LIBRARIES = libkvspool.a
libkvspool_a_SOURCES = kvspool.c kvspoolw.c kvspoolr.c kvspool_attrition.c tpl.c
include_HEADERS = ../include/kvspool.h

0
kvspool.c → src/kvspool.c
View File

0
kvspool_attrition.c → src/kvspool_attrition.c
View File

2
kvspoolr.c → src/kvspoolr.c
View File

@@ -331,7 +331,7 @@ void kv_spoolreader_free(void *_sp) {
free(sp);
}
void sp_reset(const char *dir) {
void sp_rewind(const char *dir) {
char *path, **p;
int sr_fd,rp;

0
kvspoolw.c → src/kvspoolw.c
View File

0
tpl.c → src/tpl.c
View File

2
utils/.gitignore vendored
View File

@@ -5,7 +5,7 @@ kvsp-pub
kvsp-speed
kvsp-status
kvsp-mod
kvsp-reset
kvsp-rewind
kvsp-spr
kvsp-spw
*.o

26
utils/Makefile.am
View File

@@ -1,7 +1,7 @@
AM_CFLAGS = -I.. -I../include
LIBSPOOL = -L.. -lkvspool
LIBSPOOL = -L../src -lkvspool
bin_PROGRAMS = kvsp-spr kvsp-spw kvsp-tee kvsp-size kvsp-status \
kvsp-speed kvsp-mod kvsp-reset \
kvsp-speed kvsp-mod kvsp-rewind \
ramdisk
kvsp_spr_LDADD = $(LIBSPOOL)
@@ -11,7 +11,7 @@ kvsp_size_LDADD = $(LIBSPOOL)
kvsp_status_LDADD = $(LIBSPOOL)
kvsp_speed_LDADD = $(LIBSPOOL)
kvsp_mod_LDADD = $(LIBSPOOL)
kvsp_reset_LDADD = $(LIBSPOOL)
kvsp_rewind_LDADD = $(LIBSPOOL)
kvsp_pub_LDADD = $(LIBSPOOL)
kvsp_sub_LDADD = $(LIBSPOOL)
@@ -24,13 +24,13 @@ endif
endif
# to get a rebuild of the utilities when ../libkvspool.a changes:
kvsp_spr_DEPENDENCIES = ../libkvspool.a
kvsp_spw_DEPENDENCIES = ../libkvspool.a
kvsp_tee_DEPENDENCIES = ../libkvspool.a
kvsp_size_DEPENDENCIES = ../libkvspool.a
kvsp_status_DEPENDENCIES = ../libkvspool.a
kvsp_speed_DEPENDENCIES = ../libkvspool.a
kvsp_mod_DEPENDENCIES = ../libkvspool.a
kvsp_reset_DEPENDENCIES = ../libkvspool.a
kvsp_sub_DEPENDENCIES = ../libkvspool.a
kvsp_pub_DEPENDENCIES = ../libkvspool.a
kvsp_spr_DEPENDENCIES = ../src/libkvspool.a
kvsp_spw_DEPENDENCIES = ../src/libkvspool.a
kvsp_tee_DEPENDENCIES = ../src/libkvspool.a
kvsp_size_DEPENDENCIES = ../src/libkvspool.a
kvsp_status_DEPENDENCIES = ../src/libkvspool.a
kvsp_speed_DEPENDENCIES = ../src/libkvspool.a
kvsp_mod_DEPENDENCIES = ../src/libkvspool.a
kvsp_rewind_DEPENDENCIES = ../src/libkvspool.a
kvsp_sub_DEPENDENCIES = ../src/libkvspool.a
kvsp_pub_DEPENDENCIES = ../src/libkvspool.a

2
utils/kvsp-reset.c → utils/kvsp-rewind.c
View File

@@ -24,7 +24,7 @@ int main(int argc, char *argv[]) {
if (optind < argc) dir=argv[optind++];
else usage(exe);
sp_reset(dir);
sp_rewind(dir);
return 0;
}