This commit is contained in:
Ivan
2022-04-05 11:42:28 +03:00
commit 6dc0eb0fcf
5565 changed files with 1200500 additions and 0 deletions

View File

@@ -0,0 +1,70 @@
/* This file is part of the Pangolin Project.
* http://github.com/stevenlovegrove/Pangolin
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#pragma once
#include <mutex>
#include <pangolin/log/packetstream.h>
#include <pangolin/log/packetstream_source.h>
namespace pangolin {
// Encapsulate serialized reading of Packet from stream.
struct Packet
{
Packet(PacketStream& s, std::unique_lock<std::recursive_mutex>&& mutex, std::vector<PacketStreamSource>& srcs);
Packet(const Packet&) = delete;
Packet(Packet&& o);
~Packet();
size_t BytesRead() const;
int BytesRemaining() const;
PacketStream& Stream()
{
return _stream;
}
PacketStreamSourceId src;
int64_t time;
size_t size;
size_t sequence_num;
picojson::value meta;
std::streampos frame_streampos;
private:
void ParsePacketHeader(PacketStream& s, std::vector<PacketStreamSource>& srcs);
void ReadRemaining();
PacketStream& _stream;
std::unique_lock<std::recursive_mutex> lock;
std::streampos data_streampos;
size_t _data_len;
};
}

View File

@@ -0,0 +1,111 @@
/* This file is part of the Pangolin Project.
* http://github.com/stevenlovegrove/Pangolin
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#pragma once
#include <fstream>
#include <pangolin/platform.h>
#include <pangolin/log/packetstream_tags.h>
#include <pangolin/utils/file_utils.h>
namespace pangolin
{
class PacketStream: public std::ifstream
{
public:
PacketStream()
: _is_pipe(false)
{
cclear();
}
PacketStream(const std::string& filename)
: Base(filename.c_str(), std::ios::in | std::ios::binary),
_is_pipe(IsPipe(filename))
{
cclear();
}
bool seekable() const
{
return is_open() && !_is_pipe;
}
void open(const std::string& filename)
{
close();
_is_pipe = IsPipe(filename);
Base::open(filename.c_str(), std::ios::in | std::ios::binary);
}
void close()
{
cclear();
if (Base::is_open()) Base::close();
}
void seekg(std::streampos target);
void seekg(std::streamoff off, std::ios_base::seekdir way);
std::streampos tellg();
size_t read(char* target, size_t len);
char get();
size_t skip(size_t len);
size_t readUINT();
int64_t readTimestamp();
pangoTagType peekTag();
pangoTagType readTag();
pangoTagType readTag(pangoTagType);
pangoTagType syncToTag();
private:
using Base = std::ifstream;
bool _is_pipe;
pangoTagType _tag;
// Amount of frame data left to read. Tracks our position within a data block.
void cclear() {
_tag = 0;
}
};
}

View File

@@ -0,0 +1,120 @@
/* This file is part of the Pangolin Project.
* http://github.com/stevenlovegrove/Pangolin
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#pragma once
#include <fstream>
#include <mutex>
#include <thread>
#include <pangolin/log/packet.h>
#include <pangolin/log/sync_time.h>
#include <pangolin/utils/file_utils.h>
#include <pangolin/utils/timer.h>
namespace pangolin
{
class PANGOLIN_EXPORT PacketStreamReader
{
public:
PacketStreamReader();
PacketStreamReader(const std::string& filename);
~PacketStreamReader();
void Open(const std::string& filename);
void Close();
const std::vector<PacketStreamSource>&
Sources() const
{
return _sources;
}
// Grab Next available frame packetstream
Packet NextFrame();
// Grab Next available frame in packetstream from src, discarding other frames.
Packet NextFrame(PacketStreamSourceId src);
bool Good() const
{
return _stream.good();
}
// Jumps to a particular packet.
size_t Seek(PacketStreamSourceId src, size_t framenum);
// Jumps to the first packet with time >= time
size_t Seek(PacketStreamSourceId src, SyncTime::TimePoint time);
void FixFileIndex();
private:
bool GoodToRead();
bool SetupIndex();
void ParseHeader();
void ParseNewSource();
bool ParseIndex();
void RebuildIndex();
void AppendIndex();
std::streampos ParseFooter();
void SkipSync();
void ReSync() {
_stream.syncToTag();
}
std::string _filename;
std::vector<PacketStreamSource> _sources;
SyncTime::TimePoint packet_stream_start;
PacketStream _stream;
std::recursive_mutex _mutex;
bool _is_pipe;
int _pipe_fd;
};
}

View File

@@ -0,0 +1,63 @@
#pragma once
#include <iostream>
#include <pangolin/platform.h>
#include <pangolin/utils/picojson.h>
namespace pangolin {
using PacketStreamSourceId = size_t;
struct PANGOLIN_EXPORT PacketStreamSource
{
struct PacketInfo
{
std::streampos pos;
int64_t capture_time;
};
PacketStreamSource()
: id(static_cast<PacketStreamSourceId>(-1)),
version(0),
data_alignment_bytes(1),
data_size_bytes(0),
next_packet_id(0)
{
}
std::streampos FindSeekLocation(size_t packet_id)
{
if(packet_id < index.size()) {
return index[packet_id].pos;
}else{
return std::streampos(-1);
}
}
int64_t NextPacketTime() const
{
if(next_packet_id < index.size()) {
return index[next_packet_id].capture_time;
}else{
return 0;
}
}
std::string driver;
size_t id;
std::string uri;
picojson::value info;
int64_t version;
int64_t data_alignment_bytes;
std::string data_definitions;
int64_t data_size_bytes;
// Index keyed by packet_id
std::vector<PacketInfo> index;
// Based on current position in stream
size_t next_packet_id;
};
}

View File

@@ -0,0 +1,46 @@
#pragma once
#include <string>
namespace pangolin {
using pangoTagType = uint32_t;
const static std::string PANGO_MAGIC = "PANGO";
const static std::string pss_src_driver = "driver";
const static std::string pss_src_id = "id";
const static std::string pss_src_info = "info";
const static std::string pss_src_uri = "uri";
const static std::string pss_src_packet = "packet";
const static std::string pss_src_version = "version";
const static std::string pss_pkt_alignment_bytes = "alignment_bytes";
const static std::string pss_pkt_definitions = "definitions";
const static std::string pss_pkt_size_bytes = "size_bytes";
const static std::string pss_pkt_format_written = "format_written";
const unsigned int TAG_LENGTH = 3;
#define PANGO_TAG(a,b,c) ( (c<<16) | (b<<8) | a)
const uint32_t TAG_PANGO_HDR = PANGO_TAG('L', 'I', 'N');
const uint32_t TAG_PANGO_MAGIC = PANGO_TAG('P', 'A', 'N');
const uint32_t TAG_PANGO_SYNC = PANGO_TAG('S', 'Y', 'N');
const uint32_t TAG_PANGO_STATS = PANGO_TAG('S', 'T', 'A');
const uint32_t TAG_PANGO_FOOTER = PANGO_TAG('F', 'T', 'R');
const uint32_t TAG_ADD_SOURCE = PANGO_TAG('S', 'R', 'C');
const uint32_t TAG_SRC_JSON = PANGO_TAG('J', 'S', 'N');
const uint32_t TAG_SRC_PACKET = PANGO_TAG('P', 'K', 'T');
const uint32_t TAG_END = PANGO_TAG('E', 'N', 'D');
#undef PANGO_TAG
inline std::string tagName(int v)
{
char b[4];
b[0] = v&0xff;
b[1] = (v>>8)&0xff;
b[2] = (v>>16)&0xff;
b[3] = 0x00;
return std::string(b);
}
}

View File

@@ -0,0 +1,173 @@
/* This file is part of the Pangolin Project.
* http://github.com/stevenlovegrove/Pangolin
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#pragma once
#include <ostream>
#include <pangolin/log/packetstream.h>
#include <pangolin/log/packetstream_source.h>
#include <pangolin/utils/file_utils.h>
#include <pangolin/utils/threadedfilebuf.h>
namespace pangolin
{
class PANGOLIN_EXPORT PacketStreamWriter
{
public:
PacketStreamWriter()
: _stream(&_buffer), _indexable(false), _open(false), _bytes_written(0)
{
_stream.exceptions(std::ostream::badbit);
}
PacketStreamWriter(const std::string& filename, size_t buffer_size = 100*1024*1024)
: _buffer(pangolin::PathExpand(filename), buffer_size), _stream(&_buffer),
_indexable(!IsPipe(filename)), _open(_stream.good()), _bytes_written(0)
{
_stream.exceptions(std::ostream::badbit);
WriteHeader();
}
~PacketStreamWriter() {
Close();
}
void Open(const std::string& filename, size_t buffer_size = 100 * 1024 * 1024)
{
Close();
_buffer.open(filename, buffer_size);
_open = _stream.good();
_bytes_written = 0;
_indexable = !IsPipe(filename);
WriteHeader();
}
void Close()
{
if (_open)
{
if (_indexable) {
WriteEnd();
}
_buffer.close();
_open = false;
}
}
// Does not write footer or index.
void ForceClose()
{
if (_open)
{
_buffer.force_close();
Close();
}
}
// Writes to the stream immediately upon add. Return source id # and writes
// source id # to argument struct
PacketStreamSourceId AddSource(PacketStreamSource& source);
// If constructor is called inline
PacketStreamSourceId AddSource(const PacketStreamSource& source);
void WriteSourcePacket(
PacketStreamSourceId src, const char* source,const int64_t receive_time_us,
size_t sourcelen, const picojson::value& meta = picojson::value()
);
// For stream read/write synchronization. Note that this is NOT the same as
// time synchronization on playback of iPacketStreams.
void WriteSync();
// Writes the end of the stream data, including the index. Does NOT close
// the underlying ostream.
void WriteEnd();
const std::vector<PacketStreamSource>& Sources() const {
return _sources;
}
bool IsOpen() const {
return _open;
}
private:
void WriteHeader();
void Write(const PacketStreamSource&);
void WriteMeta(PacketStreamSourceId src, const picojson::value& data);
threadedfilebuf _buffer;
std::ostream _stream;
bool _indexable, _open;
std::vector<PacketStreamSource> _sources;
size_t _bytes_written;
std::recursive_mutex _lock;
};
inline void writeCompressedUnsignedInt(std::ostream& writer, size_t n)
{
while (n >= 0x80)
{
writer.put(0x80 | (n & 0x7F));
n >>= 7;
}
writer.put(static_cast<unsigned char>(n));
}
inline void writeTimestamp(std::ostream& writer, int64_t time_us)
{
writer.write(reinterpret_cast<const char*>(&time_us), sizeof(decltype(time_us)));
}
inline void writeTag(std::ostream& writer, const pangoTagType tag)
{
writer.write(reinterpret_cast<const char*>(&tag), TAG_LENGTH);
}
inline picojson::value SourceStats(const std::vector<PacketStreamSource>& srcs)
{
picojson::value stat;
stat["num_sources"] = srcs.size();
stat["src_packet_index"] = picojson::array();
stat["src_packet_times"] = picojson::array();
for(auto& src : srcs) {
picojson::array pkt_index, pkt_times;
for (const PacketStreamSource::PacketInfo& frame : src.index) {
pkt_index.emplace_back(frame.pos);
pkt_times.emplace_back(frame.capture_time);
}
stat["src_packet_index"].push_back(std::move(pkt_index));
stat["src_packet_times"].push_back(std::move(pkt_times));
}
return stat;
}
}

View File

@@ -0,0 +1,48 @@
#pragma once
#include <memory>
#include <vector>
#include <pangolin/log/packetstream_reader.h>
#include <pangolin/utils/file_utils.h>
#include <pangolin/utils/registration.h>
namespace pangolin {
class Params;
class PlaybackSession
{
public:
// Singleton Instance
static std::shared_ptr<PlaybackSession> Default();
// Return thread-safe, shared instance of PacketStreamReader, providing
// serialised read for PacketStreamReader
std::shared_ptr<PacketStreamReader> Open(const std::string& filename)
{
const std::string path = SanitizePath(PathExpand(filename));
auto i = readers.find(path);
if(i == readers.end()) {
auto psr = std::make_shared<PacketStreamReader>(path);
readers[path] = psr;
return psr;
}else{
return i->second;
}
}
SyncTime& Time()
{
return time;
}
static std::shared_ptr<PlaybackSession> ChooseFromParams(const Params& params);
private:
std::map<std::string,std::shared_ptr<PacketStreamReader>> readers;
SyncTime time;
};
}

View File

@@ -0,0 +1,230 @@
/* This file is part of the Pangolin Project.
* http://github.com/stevenlovegrove/Pangolin
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#pragma once
#include <pangolin/platform.h>
#include <pangolin/utils/signal_slot.h>
#include <pangolin/utils/timer.h>
#include <atomic>
#include <condition_variable>
#include <mutex>
#include <queue>
namespace pangolin
{
// Lightweight timestamp class to allow synchronized playback from the same (or a different) stream.
// All playback functions called with the same SyncTime will be time-synchronized, and will remain synchronized on seek() if the SyncTime is passed in when seeking.
// Playback with multiple SyncTimes (on the same or different streams) should also be synced, even in different processes or systems (underlying clock sync is not required).
// However, playback with multiple SyncTimes will break on seek().
class PANGOLIN_EXPORT SyncTime
{
public:
using Clock = baseclock;
using Duration = Clock::duration;
using TimePoint = Clock::time_point;
struct SeekInterruption: std::runtime_error
{
SeekInterruption() : std::runtime_error("Time queue invalidated by seek"){}
};
SyncTime(Duration virtual_clock_offset = std::chrono::milliseconds(0))
: seeking(false)
{
SetOffset(virtual_clock_offset);
}
// No copy constructor
SyncTime(const SyncTime&) = delete;
void SetOffset(Duration virtual_clock_offset)
{
virtual_offset = virtual_clock_offset;
}
void SetClock(TimePoint virtual_now)
{
virtual_offset = virtual_now - Clock::now();
}
TimePoint TimeNow() const
{
return Clock::now() + virtual_offset;
}
TimePoint ToVirtual(TimePoint real) const
{
return real + virtual_offset;
}
TimePoint ToReal(TimePoint virt) const
{
return virt - virtual_offset;
}
void WaitUntil(TimePoint virtual_time) const
{
std::this_thread::sleep_until( ToReal(virtual_time) );
}
int64_t QueueEvent(int64_t new_event_time_us)
{
return WaitDequeueAndQueueEvent(0, new_event_time_us);
}
void DequeueEvent(int64_t event_time_us)
{
std::unique_lock<std::mutex> l(time_mutex);
auto i = std::find(time_queue_us.begin(), time_queue_us.end(), event_time_us);
PANGO_ENSURE(i != time_queue_us.end());
time_queue_us.erase(i);
queue_changed.notify_all();
}
int64_t WaitDequeueAndQueueEvent(int64_t event_time_us, int64_t new_event_time_us =0 )
{
std::unique_lock<std::mutex> l(time_mutex);
if(event_time_us) {
PANGO_ENSURE(time_queue_us.size());
// Wait until we're top the priority-queue
queue_changed.wait(l, [&](){
if(seeking) {
// Time queue will be invalidated on seek.
// Unblock without action
throw SeekInterruption();
}
return time_queue_us.back() == event_time_us;
});
// Dequeue
time_queue_us.pop_back();
}
if(new_event_time_us) {
// Add the new event whilst we still hold the lock, so that our
// event can't be missed
insert_sorted(time_queue_us, new_event_time_us, std::greater<int64_t>());
if(time_queue_us.back() == new_event_time_us) {
// Return to avoid yielding when we're next.
return new_event_time_us;
}
}
// Only yield if another device is next
queue_changed.notify_all();
return new_event_time_us;
}
void NotifyAll()
{
queue_changed.notify_all();
}
std::mutex& TimeMutex()
{
return time_mutex;
}
void Stop()
{
seeking = true;
OnTimeStop();
queue_changed.notify_all();
}
void Start()
{
OnTimeStart();
seeking=false;
}
void Seek(TimePoint t)
{
Stop();
OnSeek(t);
Start();
}
Signal<> OnTimeStart;
Signal<> OnTimeStop;
Signal<TimePoint> OnSeek;
private:
template< typename T, typename Pred >
static typename std::vector<T>::iterator
insert_sorted( std::vector<T> & vec, T const& item, Pred pred )
{
return vec.insert (
std::upper_bound( vec.begin(), vec.end(), item, pred ), item
);
}
std::vector<int64_t> time_queue_us;
Duration virtual_offset;
std::mutex time_mutex;
std::condition_variable queue_changed;
bool seeking;
};
struct SyncTimeEventPromise
{
SyncTimeEventPromise(SyncTime& sync, int64_t time_us = 0)
: sync(sync), time_us(time_us)
{
sync.QueueEvent(time_us);
}
~SyncTimeEventPromise()
{
Cancel();
}
void Cancel()
{
if(time_us) {
sync.DequeueEvent(time_us);
time_us = 0;
}
}
void WaitAndRenew(int64_t new_time_us)
{
time_us = sync.WaitDequeueAndQueueEvent(time_us, new_time_us);
}
private:
SyncTime& sync;
int64_t time_us;
};
}