mirror of
https://github.com/JHUAPL/kvspool.git
synced 2026-01-08 22:27:56 -05:00
work in progress
This commit is contained in:
@@ -1,9 +1,4 @@
|
|||||||
AM_CFLAGS = -fPIC -I./include
|
SUBDIRS = src utils
|
||||||
lib_LIBRARIES = libkvspool.a
|
|
||||||
libkvspool_a_SOURCES = kvspool.c kvspoolw.c kvspoolr.c kvspool_attrition.c tpl.c
|
|
||||||
include_HEADERS = kvspool.h
|
|
||||||
|
|
||||||
SUBDIRS = . utils
|
|
||||||
|
|
||||||
if HAVE_PYTHON
|
if HAVE_PYTHON
|
||||||
SUBDIRS += kvpy
|
SUBDIRS += kvpy
|
||||||
|
|||||||
8
README
8
README
@@ -1,11 +1,7 @@
|
|||||||
kvspool data streaming utilities
|
kvspool data streaming utilities
|
||||||
by Troy D. Hanson
|
by Troy D. Hanson
|
||||||
|
|
||||||
|
Documentation for kvspool is at:
|
||||||
|
|
||||||
http://troydhanson.github.com/kvspool
|
http://troydhanson.github.com/kvspool
|
||||||
|
|
||||||
Special thanks to:
|
|
||||||
|
|
||||||
Trevor Adams
|
|
||||||
JT Halbert
|
|
||||||
Jeff James
|
|
||||||
|
|
||||||
|
|||||||
10
TODO
10
TODO
@@ -1,10 +0,0 @@
|
|||||||
4. Add documentation/slides
|
|
||||||
6. Change Python to have object wrapper
|
|
||||||
|
|
||||||
#done
|
|
||||||
2. Add missing KVJava
|
|
||||||
3. Move sysutils to their own repo
|
|
||||||
5. Remove 'base' parameter everywhere
|
|
||||||
7. zcon and zcontrol should be in a ZeroMQ add-on's repo
|
|
||||||
8. kvsp-import/export should be subsumed by kvsp-pub/sub
|
|
||||||
9. put libts in external snippets repo
|
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
AC_PREREQ(2.59)
|
AC_PREREQ(2.59)
|
||||||
|
|
||||||
AC_INIT([kvspool], [1.0], [tdh@tkhanson.net])
|
AC_INIT([kvspool], [1.0], [tdh@tkhanson.net])
|
||||||
AC_CONFIG_SRCDIR(kvspool.c)
|
AC_CONFIG_SRCDIR(src/kvspool.c)
|
||||||
AC_CONFIG_AUX_DIR(config)
|
AC_CONFIG_AUX_DIR(config)
|
||||||
AC_CONFIG_HEADERS(config/config.h)
|
AC_CONFIG_HEADERS(config/config.h)
|
||||||
AM_INIT_AUTOMAKE
|
AM_INIT_AUTOMAKE
|
||||||
@@ -24,6 +24,6 @@ AM_CONDITIONAL(HAVE_PYTHON,test "x$pythonexists" = "xyes")
|
|||||||
AC_CHECK_PROG(PERL,perl,perl)
|
AC_CHECK_PROG(PERL,perl,perl)
|
||||||
AM_CONDITIONAL(HAVE_PERL,test "x$PERL" != "x")
|
AM_CONDITIONAL(HAVE_PERL,test "x$PERL" != "x")
|
||||||
|
|
||||||
AC_CONFIG_FILES(Makefile utils/Makefile)
|
AC_CONFIG_FILES(Makefile src/Makefile utils/Makefile)
|
||||||
AC_OUTPUT
|
AC_OUTPUT
|
||||||
|
|
||||||
|
|||||||
8
doc/CREDITS
Normal file
8
doc/CREDITS
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
kvspool was developed in 2011 by Troy D. Hanson
|
||||||
|
|
||||||
|
Special thanks to:
|
||||||
|
JHU/APL OTT
|
||||||
|
Trevor Adams
|
||||||
|
JT Halbert
|
||||||
|
Jeff James
|
||||||
|
Nick Clote
|
||||||
31
doc/LICENSE
Normal file
31
doc/LICENSE
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
LICENSE AND DISCLAIMER
|
||||||
|
|
||||||
|
Copyright (c) 2011 The Johns Hopkins University/Applied Physics Laboratory
|
||||||
|
|
||||||
|
This software was developed at The Johns Hopkins University/Applied Physics
|
||||||
|
Laboratory (“JHU/APL”) that is the author thereof under the “work made for
|
||||||
|
hire” provisions of the copyright law. Permission is hereby granted, free of
|
||||||
|
charge, to any person obtaining a copy of this software and associated
|
||||||
|
documentation (the “Software”), to use the Software without restriction,
|
||||||
|
including without limitation the rights to copy, modify, merge, publish,
|
||||||
|
distribute, sublicense, and/or sell copies of the Software, and to permit
|
||||||
|
others to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
1. This LICENSE AND DISCLAIMER, including the copyright notice, shall be
|
||||||
|
included in all copies of the Software, including copies of substantial
|
||||||
|
portions of the Software;
|
||||||
|
|
||||||
|
2. JHU/APL assumes no obligation to provide support of any kind with regard
|
||||||
|
to the Software. This includes no obligation to provide assistance in using
|
||||||
|
the Software nor to provide updated versions of the Software; and
|
||||||
|
|
||||||
|
3. THE SOFTWARE AND ITS DOCUMENTATION ARE PROVIDED AS IS AND WITHOUT ANY
|
||||||
|
EXPRESS OR IMPLIED WARRANTIES WHATSOEVER. ALL WARRANTIES INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, PERFORMANCE, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
|
||||||
|
AND NONINFRINGEMENT ARE HEREBY DISCLAIMED. USERS ASSUME THE ENTIRE RISK AND
|
||||||
|
LIABILITY OF USING THE SOFTWARE. USERS ARE ADVISED TO TEST THE SOFTWARE
|
||||||
|
THOROUGHLY BEFORE RELYING ON IT. IN NO EVENT SHALL THE JOHNS HOPKINS
|
||||||
|
UNIVERSITY BE LIABLE FOR ANY DAMAGES WHATSOEVER, INCLUDING, WITHOUT
|
||||||
|
LIMITATION, ANY LOST PROFITS, LOST SAVINGS OR OTHER INCIDENTAL OR
|
||||||
|
CONSEQUENTIAL DAMAGES, ARISING OUT OF THE USE OR INABILITY TO USE THE
|
||||||
|
SOFTWARE.”
|
||||||
62
doc/future.txt
Normal file
62
doc/future.txt
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
Design concepts for "v2" rewrite of kvspool
|
||||||
|
-------------------------------------------
|
||||||
|
1. Support multi-writer, multi-reader from same spool
|
||||||
|
2. Use a memory-mapped file for reading/writing spool data so that:
|
||||||
|
(1) I/O occurs through shared memory even without a ramdisk, while
|
||||||
|
(2) data is still persisted back to disk
|
||||||
|
3. Support for multi-writers requires a synchronization mechanism.
|
||||||
|
(1) This is one of the functions of the "control file".
|
||||||
|
(a) this file exists alongside the spool data file
|
||||||
|
(b) by flock'ing it (or fcntl lock on a region of it), one writer
|
||||||
|
can gain exclusive write (which applies to the spool data file too);
|
||||||
|
a second level of record-locking using fcntl lock on the spool data
|
||||||
|
file can act as a redundant safeguard
|
||||||
|
(c) the control file has the min and max sequence number in it
|
||||||
|
(d) the max sequence number is just the "frame number" of the next
|
||||||
|
frame to be written
|
||||||
|
(e) the min sequence number is incremented (sometimes by an increment
|
||||||
|
greater than one) when the writer is overwriting previous frame(s).
|
||||||
|
It's purpose is explained under the "Support for multi-readers" later.
|
||||||
|
(f) The offset of the min and max frames are also stored
|
||||||
|
(g) The control block may also contain a few time-series on write rates.
|
||||||
|
(i) It would also be possible to place the control file into the data
|
||||||
|
spool itself, in which case its a "control block" of fixed size
|
||||||
|
at the beginning; this would eliminate some failure modes and
|
||||||
|
reduce the file descriptor bookkeeping by one
|
||||||
|
4. Spool data file is a single, large, circular data buffer
|
||||||
|
(a) It is pre-created prior to data being written to reserve the space
|
||||||
|
(b) This requires that it be a non-sparse file
|
||||||
|
(c) It is used as a cyclic buffer
|
||||||
|
(d) When the end is reached, a new frame may not quite fit at the end,
|
||||||
|
in which case the frame starts at the beginning of the file; but
|
||||||
|
this requires that the frame's content-length may differ from its
|
||||||
|
stored length (so that the frame that ends up at the end of the
|
||||||
|
buffer can be adjusted to consume the full remaining space).
|
||||||
|
(e) Thus the frame format is
|
||||||
|
(1) sequence number
|
||||||
|
(2) storage length
|
||||||
|
(3) content length
|
||||||
|
(4) data (in JSON)
|
||||||
|
(f) The single large data file replaces the kvspool-v1 approach
|
||||||
|
where ten sequenced files contain the spool data, and old files
|
||||||
|
are deleted as new files are written. The v1 logic requires
|
||||||
|
detection of new files in the spool, although its advantegous
|
||||||
|
in that read/write through standard (non-mmap) calls does not
|
||||||
|
swap in the entire data spool as the v2 approach may tend to do
|
||||||
|
5. Support for multi-readers
|
||||||
|
(1) since readers that are inactive for a long time may get to the point
|
||||||
|
that their next read position is potentially invalid (due to a
|
||||||
|
writer wraparound that puts a new frame into the read area),
|
||||||
|
(a) the reader that is entering the 'read' state will first
|
||||||
|
lock the control file, acquire the minimum sequence number
|
||||||
|
to see if its exceeded its own read position
|
||||||
|
(i) If it has, then the reader has experienced frame loss
|
||||||
|
and adjusts its next-read-position to the min frame
|
||||||
|
(ii) if not, the reader can then record-lock the spool
|
||||||
|
data and read the next frame
|
||||||
|
(iii) note that if the max sequence number is the same as
|
||||||
|
the read position, then the reader needs to block
|
||||||
|
(by placing inotify on the control file, unlocking
|
||||||
|
and going into a select/epoll).
|
||||||
|
(2) If reader needs persistence for its read position it should
|
||||||
|
store its own sequence number and identifier in the spool dir
|
||||||
BIN
doc/kvspool-mini.png
Normal file
BIN
doc/kvspool-mini.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
@@ -3,56 +3,96 @@ kvspool: a tool for data streams
|
|||||||
Troy D. Hanson <tdh@tkhanson.net>
|
Troy D. Hanson <tdh@tkhanson.net>
|
||||||
v0.5, February 2012
|
v0.5, February 2012
|
||||||
|
|
||||||
kvspool: a Linux-based C library used to read and write data streams
|
kvspool (a key-value spool)::
|
||||||
|
a Linux-based C library used to read and write data streams
|
||||||
made of discrete frames; with network replication, snapshot/replay,
|
made of discrete frames; with network replication, snapshot/replay,
|
||||||
bounded disk consumption and using key-value sets as the streaming unit.
|
bounded disk consumption and using key-value sets as the streaming unit.
|
||||||
|
|
||||||
kvspool's niche
|
kvspool's niche
|
||||||
---------------
|
---------------
|
||||||
Kvspool falls somewhere between the Unix pipe, a record-oriented database and a
|
Kvspool falls somewhere between the Unix pipe, a file-backed queue and a message-passing
|
||||||
message-passing library. It's not any of those, and yet it resembles them all. It's simple
|
library. It's not quite any of them, yet resembles them all. It reflects a personal set
|
||||||
to use, but reflects a particular set of design goals.
|
of design goals.
|
||||||
|
|
||||||
* like the Unix pipe, it streams data from one program to another
|
* the spool is used to "stream" (transmit) data frames from one program to another
|
||||||
* its data capacity is configurable
|
* the spool frames are each a "hash"- a set of key-value pairs (aka a dictionary)
|
||||||
* the data is disk or ramdisk-resident
|
* the spool writer never blocks, even if the reader is slow, absent, or crashes
|
||||||
* the data stream is composed of distinct messages (or "frames")
|
* the spool is a disk- or ramdisk-resident buffer of a configurable size
|
||||||
* each "frame" is a set of key-value pairs (aka a "dictionary", "hash", etc)
|
* the spool reader gets frames from the writer via the file system only
|
||||||
* the data stream can be copied off to a "snapshot" at any time
|
* the spool reader can exit, restart, and "catch up" with the writer
|
||||||
* the data stream or snapshot supports rewind and replay
|
* the spool reader blocks (waiting for new data) when its caught up
|
||||||
* streams can be sent over a network
|
* the spool reader loses data if its absent/offline/slow enough
|
||||||
|
* the spool frames remain on disk til their space is reclaimed
|
||||||
|
* the spool can be copied off to a "snapshot" at any time
|
||||||
|
* the spool supports rewind and replay
|
||||||
|
* the spool can be sent over a network
|
||||||
|
|
||||||
|
Sneak peak
|
||||||
|
~~~~~~~~~~
|
||||||
|
Here's an example of writing from Perl and reading from C.
|
||||||
|
|
||||||
|
[options="header"]
|
||||||
|
|===============================================================================
|
||||||
|
| Perl writer | C reader
|
||||||
|
| use KVSpool; | #include "kvspool.h"
|
||||||
|
| my $v = KVSpool->new("spool"); | void *sp = kv_spoolreader_new("spool");
|
||||||
|
| my $h = {'day'=>'Wed','temp'=>37}; | void *set = kv_set_new();
|
||||||
|
| $v->write($h); | kv_spool_read(sp,set,1);
|
||||||
|
|===============================================================================
|
||||||
|
|
||||||
|
Rewind and replay
|
||||||
|
~~~~~~~~~~~~~~~~~
|
||||||
|
Kvspool keeps the data frames, even after they've been read-- til space needs to be
|
||||||
|
reclaimed. (So the spool is a like a long reel of tape spliced together at the ends).
|
||||||
|
There are several nice outcomes of this:
|
||||||
|
|
||||||
|
* You have a history, or a "rear-view window" of the stream from writer to reader
|
||||||
|
* Because you have this history of the stream, you can copy it off
|
||||||
|
* You can take it back to a development or test environment
|
||||||
|
* You can "rewind" and "replay" the spool for testing
|
||||||
|
|
||||||
|
Canned data
|
||||||
|
^^^^^^^^^^^
|
||||||
|
For developers, kvspool can be a convenient way to take "canned" data from a production
|
||||||
|
environment. Just copy the spool. Now the data is canned. The developer can now take it
|
||||||
|
on a laptop (where the writer is not even necessary), rewind it, and use it as input.
|
||||||
|
|
||||||
Platform
|
Platform
|
||||||
~~~~~~~~
|
~~~~~~~~
|
||||||
Kvspool is written for Linux, and has support for C, Perl, Python and Java.
|
Kvspool is written for Linux, and has support for C, Perl, Python and Java.
|
||||||
The C library does not depend on any other libraries; if you have ZeroMQ (2.x
|
While the C library does not depend on any other libraries, it's recommended to have if
|
||||||
or 3.x) installed and the Jansson library installed, additional utilities for network
|
you have *ZeroMQ* (2.x or 3.x) and the *Jansson* library installed, additional utilities for
|
||||||
replication of spools are built.
|
network replication of spools are built.
|
||||||
|
|
||||||
License
|
License
|
||||||
~~~~~~~
|
~~~~~~~
|
||||||
See the LICENSE file. Kvspool is free and open source.
|
See the link:LICENSE[LICENSE] file. Kvspool is free and open source.
|
||||||
|
|
||||||
Motivation
|
Resources & Help
|
||||||
~~~~~~~~~~
|
~~~~~~~~~~~~~~~~
|
||||||
It all started with a sensor. Like any sensor this one produced an endless series of
|
News about software updates are posted to the author's blog: http://tkhanson.net/blog.
|
||||||
|
Contact the author directly at tdh@tkhanson.net if you have questions or other issues.
|
||||||
|
|
||||||
|
History & Motivation
|
||||||
|
~~~~~~~~~~~~~~~~~~~~
|
||||||
|
It started with a sensor. Like any sensor this one produced an endless series of
|
||||||
measurements. The measurements were fed into another process. How? With a Unix pipe:
|
measurements. The measurements were fed into another process. How? With a Unix pipe:
|
||||||
|
|
||||||
sensor | analyzer
|
sensor | analyzer
|
||||||
|
|
||||||
Beatiful and concise, but:
|
Beatiful and concise, but:
|
||||||
|
|
||||||
* what happens if 'sensor' produces data faster than 'analyzer' can read it?
|
* what happens if `sensor` produces data faster than `analyzer` can read it?
|
||||||
* What happens to 'sensor' if 'analyzer' crashes?
|
* What happens to `sensor` if `analyzer` crashes?
|
||||||
|
|
||||||
If 'sensor' is doing something important- the pipe is not robust because 'sensor' gets
|
If `sensor` is doing something important- the pipe is not robust because `sensor` gets
|
||||||
'blocked' (put to sleep) if 'analyzer' reads the pipe too slowly-- and worse yet,
|
blocked (put to sleep) if 'analyzer' reads the pipe too slowly-- and worse yet,
|
||||||
any bugs that crash 'analyzer' thereby break the pipe and crash 'sensor' too.
|
any bugs that crash '`nalyzer` thereby break the pipe and crash `sensor` too.
|
||||||
|
|
||||||
In search of
|
In search of
|
||||||
^^^^^^^^^^^^
|
^^^^^^^^^^^^
|
||||||
The `sensor | analyzer` pipeline could be replaced many ways: for example 'sensor' could
|
The `sensor | analyzer` pipeline could be replaced many ways: for example `sensor` could
|
||||||
write to a database, which 'analyzer' could poll periodically. But Unix people dislike
|
write to a database, which `analyzer` could poll periodically. But Unix people dislike
|
||||||
polling. It says "I couldn't figure out an event-driven solution to this problem". We
|
polling. It says "I couldn't figure out an event-driven solution to this problem". We
|
||||||
could use shared memory, and semaphores, etc. However, there's also a Unix mindset that
|
could use shared memory, and semaphores, etc. However, there's also a Unix mindset that
|
||||||
says "everything is a file"-- so why shouldn't our data stream be one too? In other
|
says "everything is a file"-- so why shouldn't our data stream be one too? In other
|
||||||
@@ -61,18 +101,18 @@ benefits that confers (for example, the ability to copy it easily) and yet still
|
|||||||
event-driven model where the reader is woken up only when new data is available? (Yes, we
|
event-driven model where the reader is woken up only when new data is available? (Yes, we
|
||||||
can, using inotify). The wish list became,
|
can, using inotify). The wish list became,
|
||||||
|
|
||||||
#. stream should be a file (can be on a ram disk)
|
* stream should be a file (can be on a ram disk)
|
||||||
#. reader should be event driven
|
* reader should be event driven
|
||||||
#. let the user configure how much disk space to allocate to the stream
|
* let the user configure how much disk space to allocate to the stream
|
||||||
#. drop old data (whether its read or unread) when the stream fills up
|
* drop old data (whether its read or unread) when the stream fills up
|
||||||
#. put framing into the stream so that we can read and write whole messages
|
* put framing into the stream so that we can read and write whole messages
|
||||||
#. use key-value sets (aka a dictionary or hash) as the data unit
|
* use key-value sets (aka a dictionary or hash) as the data unit
|
||||||
#. copy a "live" data stream to a frozen "snapshot"
|
* copy a "live" data stream to a frozen "snapshot"
|
||||||
#. support rewind and replay.
|
* support rewind and replay.
|
||||||
#. work locally or over a network.
|
* work locally or over a network.
|
||||||
#. insulate writer from reader (so much that either can be absent or sporadically present)
|
* insulate writer from reader (so much that either can be absent or sporadically present)
|
||||||
#. work with many languages.
|
* work with many languages.
|
||||||
#. easy to use.
|
* easy to use.
|
||||||
|
|
||||||
Kvspool does these things. It's not a sophisticated suite. It's just a tool in the
|
Kvspool does these things. It's not a sophisticated suite. It's just a tool in the
|
||||||
Unix tradition that does one thing and tries to do it well.
|
Unix tradition that does one thing and tries to do it well.
|
||||||
@@ -85,12 +125,6 @@ or rewind a stream, to watch its status, set up network replication, and so on.
|
|||||||
used to read and write the stream is extremely simple: key-value sets (dictionary or hash
|
used to read and write the stream is extremely simple: key-value sets (dictionary or hash
|
||||||
are common names for this data structure) are simply read from the stream or written to it.
|
are common names for this data structure) are simply read from the stream or written to it.
|
||||||
|
|
||||||
The best feature
|
|
||||||
^^^^^^^^^^^^^^^^
|
|
||||||
Perhaps more than any other feature, the ability to copy a "live" data stream to an
|
|
||||||
offline "snapshot", and then replay it as often as necessary, for testing or algorithm
|
|
||||||
development, can be very valuable. Testing with a consistent data set becomes easy.
|
|
||||||
|
|
||||||
Does kvspool keep data after its been read?
|
Does kvspool keep data after its been read?
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
Yes, for two reasons. Kvspool keeps data, even after its been read, up to the maximum
|
Yes, for two reasons. Kvspool keeps data, even after its been read, up to the maximum
|
||||||
@@ -116,11 +150,6 @@ To build it:
|
|||||||
This builds and installs the C library and utilities, and if the prerequisite packages
|
This builds and installs the C library and utilities, and if the prerequisite packages
|
||||||
are installed, it builds the Perl, Python and Java bindings, and ZeroMQ-based utilities.
|
are installed, it builds the Perl, Python and Java bindings, and ZeroMQ-based utilities.
|
||||||
|
|
||||||
Resources & Help
|
|
||||||
~~~~~~~~~~~~~~~~
|
|
||||||
News about software updates are posted to the author's blog: http://tkhanson.net/blog.
|
|
||||||
Contact the author directly at tdh@tkhanson.net if you have questions or other issues.
|
|
||||||
|
|
||||||
Basics
|
Basics
|
||||||
------
|
------
|
||||||
|
|
||||||
@@ -146,17 +175,20 @@ Write data to spool
|
|||||||
We show a simple example of using the spool in Perl, Python and C here.
|
We show a simple example of using the spool in Perl, Python and C here.
|
||||||
|
|
||||||
.Perl
|
.Perl
|
||||||
|
[source,perl]
|
||||||
use KVSpool;
|
use KVSpool;
|
||||||
my $h = {'day' => 'Wednesday', 'user' => 'Troy'};
|
my $h = {'day' => 'Wednesday', 'user' => 'Troy'};
|
||||||
my $v = KVSpool->new("spool");
|
my $v = KVSpool->new("spool");
|
||||||
$v->write($h);
|
$v->write($h);
|
||||||
|
|
||||||
.Python
|
.Python
|
||||||
|
[source,python]
|
||||||
import kvpy
|
import kvpy
|
||||||
d = {"day":"Wednesday","user":"Troy"}
|
d = {"day":"Wednesday","user":"Troy"}
|
||||||
kvpy.kvpy_write("spool",d)
|
kvpy.kvpy_write("spool",d)
|
||||||
|
|
||||||
.C
|
.C
|
||||||
|
[source,c]
|
||||||
#include "kvspool.h"
|
#include "kvspool.h"
|
||||||
...
|
...
|
||||||
void *sp = kv_spoolwriter_new("spool");
|
void *sp = kv_spoolwriter_new("spool");
|
||||||
@@ -170,15 +202,18 @@ We show a simple example of using the spool in Perl, Python and C here.
|
|||||||
Read data from spool
|
Read data from spool
|
||||||
~~~~~~~~~~~~~~~~~~~~
|
~~~~~~~~~~~~~~~~~~~~
|
||||||
.Perl
|
.Perl
|
||||||
|
[source,perl]
|
||||||
use KVSpool;
|
use KVSpool;
|
||||||
my $v = KVSpool->new("spool");
|
my $v = KVSpool->new("spool");
|
||||||
my $h = $v->read();
|
my $h = $v->read();
|
||||||
|
|
||||||
.Python
|
.Python
|
||||||
|
[source,python]
|
||||||
import kvpy
|
import kvpy
|
||||||
d = kvpy.kvpy_read("spool")
|
d = kvpy.kvpy_read("spool")
|
||||||
|
|
||||||
.C
|
.C
|
||||||
|
[source,c]
|
||||||
#include "kvspool.h"
|
#include "kvspool.h"
|
||||||
...
|
...
|
||||||
void *sp = kv_spoolreader_new("spool");
|
void *sp = kv_spoolreader_new("spool");
|
||||||
@@ -228,7 +263,7 @@ is easy to overcome when multiple independent readers each need their own copy o
|
|||||||
Persistent read position
|
Persistent read position
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
The spool records the reader position internally. If a reader exits, then restarts, it
|
The spool records the reader position internally. If a reader exits, then restarts, it
|
||||||
picks up where it left off. (The `kvsp-reset` utility can be used to reset the reader
|
picks up where it left off. (The `kvsp-rewind` utility can be used to reset the reader
|
||||||
position to the beginning, for replay purposes).
|
position to the beginning, for replay purposes).
|
||||||
|
|
||||||
Because the read position is stored in the spool, you can see it using `kvsp-status`.
|
Because the read position is stored in the spool, you can see it using `kvsp-status`.
|
||||||
@@ -257,12 +292,14 @@ Programs written against the kvspool API can be linked with -lkvspool.
|
|||||||
|
|
||||||
Reader API
|
Reader API
|
||||||
^^^^^^^^^^
|
^^^^^^^^^^
|
||||||
|
[source,c]
|
||||||
void *kv_spoolreader_new(const char *dir);
|
void *kv_spoolreader_new(const char *dir);
|
||||||
int kv_spool_read(void*sp, void *set, int blocking);
|
int kv_spool_read(void*sp, void *set, int blocking);
|
||||||
void kv_spoolreader_free(void*);
|
void kv_spoolreader_free(void*);
|
||||||
|
|
||||||
Writer API
|
Writer API
|
||||||
^^^^^^^^^^
|
^^^^^^^^^^
|
||||||
|
[source,c]
|
||||||
void *kv_spoolwriter_new(const char *dir);
|
void *kv_spoolwriter_new(const char *dir);
|
||||||
int kv_spool_write(void*sp, void *set);
|
int kv_spool_write(void*sp, void *set);
|
||||||
void kv_spoolwriter_free(void*);
|
void kv_spoolwriter_free(void*);
|
||||||
@@ -271,6 +308,7 @@ Dictionary API
|
|||||||
^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^
|
||||||
The `void *set` in the C API is a dictionary data structure in C.
|
The `void *set` in the C API is a dictionary data structure in C.
|
||||||
|
|
||||||
|
[source,c]
|
||||||
void* kv_set_new(void);
|
void* kv_set_new(void);
|
||||||
void kv_set_free(void*);
|
void kv_set_free(void*);
|
||||||
void kv_set_clear(void*);
|
void kv_set_clear(void*);
|
||||||
@@ -291,6 +329,7 @@ The `void *set` in the C API is a dictionary data structure in C.
|
|||||||
|
|
||||||
A C program can iterate through all the keys/values like:
|
A C program can iterate through all the keys/values like:
|
||||||
|
|
||||||
|
[source,c]
|
||||||
kv_t *kv = NULL;
|
kv_t *kv = NULL;
|
||||||
while ( (kv = kv_next(set, kv))) {
|
while ( (kv = kv_next(set, kv))) {
|
||||||
printf("key is %s\n", kv->key);
|
printf("key is %s\n", kv->key);
|
||||||
@@ -299,23 +338,27 @@ A C program can iterate through all the keys/values like:
|
|||||||
|
|
||||||
Reset API
|
Reset API
|
||||||
~~~~~~~~~
|
~~~~~~~~~
|
||||||
This is the programmatic equal of the `kvsp-reset` command:
|
This is the programmatic equal of the `kvsp-rewind` command:
|
||||||
|
|
||||||
|
[source,c]
|
||||||
void sp_reset(const char *dir);
|
void sp_reset(const char *dir);
|
||||||
|
|
||||||
Perl
|
Perl
|
||||||
~~~~
|
~~~~
|
||||||
In Perl this is how to use the module and open a spool for reading or writing:
|
In Perl this is how to use the module and open a spool for reading or writing:
|
||||||
|
|
||||||
|
[source,perl]
|
||||||
use KVSpool;
|
use KVSpool;
|
||||||
my $v = KVSpool->new("spool");
|
my $v = KVSpool->new("spool");
|
||||||
|
|
||||||
Then to read:
|
Then to read:
|
||||||
|
|
||||||
|
[source,perl]
|
||||||
my $h = $v->read(); # returns a hash reference
|
my $h = $v->read(); # returns a hash reference
|
||||||
|
|
||||||
Similarly to write:
|
Similarly to write:
|
||||||
|
|
||||||
|
[source,perl]
|
||||||
$v->write($h); # where h is a hash reference
|
$v->write($h); # where h is a hash reference
|
||||||
|
|
||||||
Python
|
Python
|
||||||
@@ -323,6 +366,7 @@ Python
|
|||||||
As of the current version kvspool only has a procedural interface for Python. If d is a
|
As of the current version kvspool only has a procedural interface for Python. If d is a
|
||||||
dicionary then the API to write or read a frame is simply:
|
dicionary then the API to write or read a frame is simply:
|
||||||
|
|
||||||
|
[source,python]
|
||||||
import kvpy
|
import kvpy
|
||||||
kvpy.kvpy_write("spool",d)
|
kvpy.kvpy_write("spool",d)
|
||||||
d = kvpy.kvpy_read("spool")
|
d = kvpy.kvpy_read("spool")
|
||||||
@@ -339,7 +383,7 @@ Basic
|
|||||||
|command | example
|
|command | example
|
||||||
|kvsp-size | kvsp-size -s 1G spool
|
|kvsp-size | kvsp-size -s 1G spool
|
||||||
|kvsp-status | kvsp-status spool
|
|kvsp-status | kvsp-status spool
|
||||||
|kvsp-reset | kvsp-reset spool
|
|kvsp-rewind | kvsp-rewind spool
|
||||||
|kvsp-tee | kvsp-tee -s spool-in spool-copy1 spool-copy2
|
|kvsp-tee | kvsp-tee -s spool-in spool-copy1 spool-copy2
|
||||||
|===============================================================================
|
|===============================================================================
|
||||||
|
|
||||||
@@ -349,7 +393,7 @@ run later, after the spool already exists and has data, it is resized.
|
|||||||
|
|
||||||
Run `kvsp-status` to see what percentage of the spool has been consumed by a reader.
|
Run `kvsp-status` to see what percentage of the spool has been consumed by a reader.
|
||||||
|
|
||||||
The `kvsp-reset` command resets the reader position to the beginning (oldest frame) in the
|
The `kvsp-rewind` command resets the reader position to the beginning (oldest frame) in the
|
||||||
spool. Use this command in order to "replay" the spooled data. Disconnect (terminate) any
|
spool. Use this command in order to "replay" the spooled data. Disconnect (terminate) any
|
||||||
readers before running this command.
|
readers before running this command.
|
||||||
|
|
||||||
@@ -453,7 +497,7 @@ To snapshot a spool, just copy it:
|
|||||||
copying the spool). With the snapshot copied off, it can now be "replayed" as often as
|
copying the spool). With the snapshot copied off, it can now be "replayed" as often as
|
||||||
needed to develop new versions of the software that reads it.
|
needed to develop new versions of the software that reads it.
|
||||||
|
|
||||||
kvsp-reset snapshot
|
kvsp-rewind snapshot
|
||||||
|
|
||||||
Since the snapshot is "canned" real data, but not being written to any longer, it is
|
Since the snapshot is "canned" real data, but not being written to any longer, it is
|
||||||
useful as a consistent data set to test new versions of software. The other major benefit
|
useful as a consistent data set to test new versions of software. The other major benefit
|
||||||
@@ -522,7 +566,7 @@ Rough edges:
|
|||||||
|
|
||||||
More sweeping ideas for a possible future "v2" rewrite:
|
More sweeping ideas for a possible future "v2" rewrite:
|
||||||
|
|
||||||
* Support multi-writer, multi-reader (see multi_future.txt)
|
* Support multi-writer, multi-reader (see future.txt)
|
||||||
* Replace segemented data files with one memory mapped, circular file
|
* Replace segemented data files with one memory mapped, circular file
|
||||||
* Use JSON internally
|
* Use JSON internally
|
||||||
|
|
||||||
@@ -51,7 +51,7 @@ void sp_attrition(char *dir);
|
|||||||
*****************************************************************************/
|
*****************************************************************************/
|
||||||
typedef struct { int pct_consumed; } kv_stat_t;
|
typedef struct { int pct_consumed; } kv_stat_t;
|
||||||
int kv_stat(const char *dir, kv_stat_t *stats);
|
int kv_stat(const char *dir, kv_stat_t *stats);
|
||||||
void sp_reset(const char *dir);
|
void sp_rewind(const char *dir);
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
size_t dir_max;
|
size_t dir_max;
|
||||||
5
src/Makefile.am
Normal file
5
src/Makefile.am
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
AM_CFLAGS = -fPIC -I../include
|
||||||
|
lib_LIBRARIES = libkvspool.a
|
||||||
|
libkvspool_a_SOURCES = kvspool.c kvspoolw.c kvspoolr.c kvspool_attrition.c tpl.c
|
||||||
|
include_HEADERS = ../include/kvspool.h
|
||||||
|
|
||||||
@@ -331,7 +331,7 @@ void kv_spoolreader_free(void *_sp) {
|
|||||||
free(sp);
|
free(sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
void sp_reset(const char *dir) {
|
void sp_rewind(const char *dir) {
|
||||||
char *path, **p;
|
char *path, **p;
|
||||||
int sr_fd,rp;
|
int sr_fd,rp;
|
||||||
|
|
||||||
2
utils/.gitignore
vendored
2
utils/.gitignore
vendored
@@ -5,7 +5,7 @@ kvsp-pub
|
|||||||
kvsp-speed
|
kvsp-speed
|
||||||
kvsp-status
|
kvsp-status
|
||||||
kvsp-mod
|
kvsp-mod
|
||||||
kvsp-reset
|
kvsp-rewind
|
||||||
kvsp-spr
|
kvsp-spr
|
||||||
kvsp-spw
|
kvsp-spw
|
||||||
*.o
|
*.o
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
AM_CFLAGS = -I.. -I../include
|
AM_CFLAGS = -I.. -I../include
|
||||||
LIBSPOOL = -L.. -lkvspool
|
LIBSPOOL = -L../src -lkvspool
|
||||||
bin_PROGRAMS = kvsp-spr kvsp-spw kvsp-tee kvsp-size kvsp-status \
|
bin_PROGRAMS = kvsp-spr kvsp-spw kvsp-tee kvsp-size kvsp-status \
|
||||||
kvsp-speed kvsp-mod kvsp-reset \
|
kvsp-speed kvsp-mod kvsp-rewind \
|
||||||
ramdisk
|
ramdisk
|
||||||
|
|
||||||
kvsp_spr_LDADD = $(LIBSPOOL)
|
kvsp_spr_LDADD = $(LIBSPOOL)
|
||||||
@@ -11,7 +11,7 @@ kvsp_size_LDADD = $(LIBSPOOL)
|
|||||||
kvsp_status_LDADD = $(LIBSPOOL)
|
kvsp_status_LDADD = $(LIBSPOOL)
|
||||||
kvsp_speed_LDADD = $(LIBSPOOL)
|
kvsp_speed_LDADD = $(LIBSPOOL)
|
||||||
kvsp_mod_LDADD = $(LIBSPOOL)
|
kvsp_mod_LDADD = $(LIBSPOOL)
|
||||||
kvsp_reset_LDADD = $(LIBSPOOL)
|
kvsp_rewind_LDADD = $(LIBSPOOL)
|
||||||
kvsp_pub_LDADD = $(LIBSPOOL)
|
kvsp_pub_LDADD = $(LIBSPOOL)
|
||||||
kvsp_sub_LDADD = $(LIBSPOOL)
|
kvsp_sub_LDADD = $(LIBSPOOL)
|
||||||
|
|
||||||
@@ -24,13 +24,13 @@ endif
|
|||||||
endif
|
endif
|
||||||
|
|
||||||
# to get a rebuild of the utilities when ../libkvspool.a changes:
|
# to get a rebuild of the utilities when ../libkvspool.a changes:
|
||||||
kvsp_spr_DEPENDENCIES = ../libkvspool.a
|
kvsp_spr_DEPENDENCIES = ../src/libkvspool.a
|
||||||
kvsp_spw_DEPENDENCIES = ../libkvspool.a
|
kvsp_spw_DEPENDENCIES = ../src/libkvspool.a
|
||||||
kvsp_tee_DEPENDENCIES = ../libkvspool.a
|
kvsp_tee_DEPENDENCIES = ../src/libkvspool.a
|
||||||
kvsp_size_DEPENDENCIES = ../libkvspool.a
|
kvsp_size_DEPENDENCIES = ../src/libkvspool.a
|
||||||
kvsp_status_DEPENDENCIES = ../libkvspool.a
|
kvsp_status_DEPENDENCIES = ../src/libkvspool.a
|
||||||
kvsp_speed_DEPENDENCIES = ../libkvspool.a
|
kvsp_speed_DEPENDENCIES = ../src/libkvspool.a
|
||||||
kvsp_mod_DEPENDENCIES = ../libkvspool.a
|
kvsp_mod_DEPENDENCIES = ../src/libkvspool.a
|
||||||
kvsp_reset_DEPENDENCIES = ../libkvspool.a
|
kvsp_rewind_DEPENDENCIES = ../src/libkvspool.a
|
||||||
kvsp_sub_DEPENDENCIES = ../libkvspool.a
|
kvsp_sub_DEPENDENCIES = ../src/libkvspool.a
|
||||||
kvsp_pub_DEPENDENCIES = ../libkvspool.a
|
kvsp_pub_DEPENDENCIES = ../src/libkvspool.a
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ int main(int argc, char *argv[]) {
|
|||||||
if (optind < argc) dir=argv[optind++];
|
if (optind < argc) dir=argv[optind++];
|
||||||
else usage(exe);
|
else usage(exe);
|
||||||
|
|
||||||
sp_reset(dir);
|
sp_rewind(dir);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
Reference in New Issue
Block a user