Implementation Details. More...

Classes
class	BulkJobMetadata

struct	ColumnCompressionOptions

class	ColumnEnumerator

class	ColumnSink

class	ColumnSource

struct	DAGAnalysisInfo

class	DatabaseMetadata

struct	DatabaseParameters

class	DecoderAutomata

class	DomainSampler

struct	EncodeOptions

class	EnumeratorBuilder

class	EnumeratorFactory
	Interface for constructing enumerators at runtime. More...

class	EnumeratorRegistration

class	EnumeratorRegistry

class	EvaluateWorker

struct	EvaluateWorkerArgs

struct	EvalWorkEntry

class	H264ByteStreamIndexCreator

class	ImageFormatGroupMetadata

class	KernelBuilder

class	KernelFactory
	Interface for constructing ops at runtime. More...

class	KernelRegistration

class	KernelRegistry

class	LoadWorker

struct	LoadWorkerArgs

class	MasterServerImpl

class	Metadata
	Common persistent data structs and their serialization helpers. More...

class	NVIDIAVideoDecoder
	NVIDIAVideoDecoder. More...

struct	OpArgGroup

class	OpBuilder

class	OpInfo

class	OpRegistration

class	OpRegistry

class	Partitioner

struct	PartitionGroup

class	PostEvaluateWorker

struct	PostEvaluateWorkerArgs

class	PreEvaluateWorker

struct	PreEvaluateWorkerArgs
	Worker thread arguments. More...

class	SamplerRegistry

class	SaveWorker

struct	SaveWorkerArgs

class	SinkBuilder

class	SinkFactory
	Interface for constructing Sinks at runtime. More...

class	SinkRegistration

class	SinkRegistry

class	SoftwareVideoDecoder
	SoftwareVideoDecoder. More...

class	SoftwareVideoEncoder
	SoftwareVideoEncoder. More...

class	SourceBuilder

class	SourceFactory
	Interface for constructing Sources at runtime. More...

class	SourceRegistration

class	SourceRegistry

class	TableMetaCache

class	TableMetadata

struct	TaskStream

class	VideoDecoder
	VideoDecoder. More...

class	VideoEncoder
	VideoEncoder. More...

struct	VideoIndexEntry

class	VideoMetadata

class	WorkerImpl

Typedefs
using	EnumeratorConstructor = std::function< Enumerator *(const EnumeratorConfig &config)>

using	KernelConstructor = std::function< BaseKernel *(const KernelConfig &config)>

using	SinkConstructor = std::function< Sink *(const SinkConfig &config)>

using	SourceConstructor = std::function< Source *(const SourceConfig &config)>

using	JobID = i64

using	WorkerID = i64

template<class Request , class Reply >
using	MCall = Call< MasterServerImpl, Request, Reply >

template<typename T >
using	WriteFn = void()(storehouse::StorageBackend storage, T db_proto)

template<typename T >
using	ReadFn = T()(storehouse::StorageBackend storage, const std::string &path)

using	LoadInputQueue = Queue< std::tuple< i32, std::deque< TaskStream >, LoadWorkEntry >>

using	EvalQueue = Queue< std::tuple< std::deque< TaskStream >, EvalWorkEntry >>

using	OutputEvalQueue = Queue< std::tuple< i32, EvalWorkEntry >>

using	SaveInputQueue = Queue< std::tuple< i32, EvalWorkEntry >>

using	SaveOutputQueue = Queue< std::tuple< i32, i64, i64 >>

Enumerations
enum	VideoDecoderType { NVIDIA, INTEL, SOFTWARE }

enum	VideoEncoderType { NVIDIA, INTEL, SOFTWARE }

Functions
	protobuf_name ("ColumnEnumeratorArgs")

	variadic_inputs ().per_element_output().protobuf_name("ColumnSinkArgs")

	output ("output").protobuf_name("ColumnSourceArgs")

	frame_output ("frame_output").protobuf_name("ColumnSourceArgs")

bool	is_builtin_op (const std::string &name)

Result	validate_jobs_and_ops (DatabaseMetadata &meta, TableMetaCache &table_metas, const std::vector< proto::Job > &jobs, const std::vector< proto::Op > &ops, DAGAnalysisInfo &info)

Result	determine_input_rows_to_slices (DatabaseMetadata &meta, TableMetaCache &table_metas, const std::vector< proto::Job > &jobs, const std::vector< proto::Op > &ops, DAGAnalysisInfo &info, storehouse::StorageConfig *storage_config)

std::tuple< i64, i64 >	determine_stencil_bounds (const std::vector< proto::Op > &ops)

Result	derive_slice_final_output_rows (const proto::Job &job, const std::vector< proto::Op > &ops, i64 slice_op_idx, i64 slice_input_rows, DAGAnalysisInfo &info, std::vector< i64 > &slice_output_partition)

void	populate_analysis_info (const std::vector< proto::Job > &jobs, const std::vector< proto::Op > &ops, DAGAnalysisInfo &info)

void	remap_input_op_edges (std::vector< proto::Op > &ops, DAGAnalysisInfo &info)

void	remap_sink_op_edges (std::vector< proto::Op > &ops, DAGAnalysisInfo &info)

void	perform_liveness_analysis (const std::vector< proto::Op > &ops, DAGAnalysisInfo &results)

Result	derive_stencil_requirements (const DatabaseMetadata &meta, TableMetaCache &table_meta, const proto::Job &job, const std::vector< proto::Op > &ops, const DAGAnalysisInfo &analysis_results, proto::BulkJobParameters::BoundaryCondition boundary_condition, i64 job_idx, i64 task_idx, const std::vector< i64 > &output_rows, LoadWorkEntry &output_entry, std::deque< TaskStream > &task_streams, storehouse::StorageConfig *storage_config)

EnumeratorRegistry *	get_enumerator_registry ()

void	move_if_different_address_space (Profiler &profiler, DeviceHandle current_handle, DeviceHandle target_handle, BatchedElements &columns)

Result	ingest_videos (storehouse::StorageConfig *storage_config, const std::string &db_path, const std::vector< std::string > &table_names, const std::vector< std::string > &paths, bool inplace, std::vector< FailedVideo > &failed_videos)

void	ingest_images (storehouse::StorageConfig *storage_config, const std::string &db_path, const std::string &table_name, const std::vector< std::string > &paths)

KernelRegistry *	get_kernel_registry ()

const std::string &	get_database_path ()
	Path functions.

void	set_database_path (std::string path)

void	write_table_megafile (storehouse::StorageBackend *storage, const std::unordered_map< i32, TableMetadata > &table_metadata)

void	read_table_megafile (storehouse::StorageBackend *storage, std::unordered_map< i32, TableMetadata > &table_metadata)

const std::string	get_scanner_path ()

void	set_scanner_path (std::string path)

std::string	database_metadata_path ()

std::string	table_megafile_path ()

std::string	table_directory (i32 table_id)

std::string	table_descriptor_path (i32 table_id)

std::string	table_item_output_path (i32 table_id, i32 column_id, i32 item_id)

std::string	table_item_video_metadata_path (i32 table_id, i32 column_id, i32 item_id)

std::string	table_item_metadata_path (i32 table_id, i32 column_id, i32 item_id)

std::string	bulk_job_directory (i32 bulk_job_id)

std::string	bulk_job_descriptor_path (i32 bulk_job_id)

std::string	bulk_job_master_profiler_path (i32 bulk_job_id)

std::string	bulk_job_worker_profiler_path (i32 bulk_job_id, i32 node)

std::string	index_column_name ()
	Constants.

std::string	frame_column_name ()

std::string	frame_info_column_name ()

template<typename T >
void	serialize_db_proto (storehouse::WriteFile *file, const T &descriptor)
	Helpers.

template<typename T >
T	deserialize_db_proto (storehouse::RandomReadFile *file, u64 &pos)

template<typename T >
void	write_db_proto (storehouse::StorageBackend *storage, T db_proto)

template<typename T >
T	read_db_proto (storehouse::StorageBackend *storage, const std::string &path)

OpRegistry *	get_op_registry ()

MasterServerImpl *	get_master_service (DatabaseParameters &param, const std::string &port)

WorkerImpl *	get_worker_service (DatabaseParameters &params, const std::string &master_address, const std::string &worker_port)

void	move_if_different_address_space (Profiler &profiler, DeviceHandle current_handle, DeviceHandle target_handle, Elements &column)

Elements	copy_elements (Profiler &profiler, DeviceHandle current_handle, DeviceHandle target_handle, Elements &column)

Elements	copy_or_ref_elements (Profiler &profiler, DeviceHandle current_handle, DeviceHandle target_handle, Elements &column)

Result	make_domain_sampler_instance (const std::string &sampler_type, const std::vector< u8 > &sampler_args, DomainSampler *&sampler)

Result	make_partitioner_instance (const std::string &sampler_type, const std::vector< u8 > &sampler_args, i64 num_rows, Partitioner *&sampler)

SinkRegistry *	get_sink_registry ()

SourceRegistry *	get_source_registry ()

VideoIndexEntry	read_video_index (storehouse::StorageBackend *storage, i32 table_id, i32 column_id, i32 item_id)

VideoIndexEntry	read_video_index (storehouse::StorageBackend *storage, const VideoMetadata &video_meta)

	TEST (DecoderAutomata, GetAllFrames)

	TEST (DecoderAutomata, GetStridedFrames)

Variables
const i32	MAX_SINK_THREADS = 16

const std::string	INPUT_OP_NAME = "Input"

const std::string	OUTPUT_OP_NAME = "OutputTable"

const std::string	SAMPLE_OP_NAME = "Sample"

const std::string	SPACE_OP_NAME = "Space"

const std::string	SLICE_OP_NAME = "Slice"

const std::string	UNSLICE_OP_NAME = "Unslice"

const std::vector< std::string >	BUILTIN_OP_NAMES

const i32	MAX_SOURCE_THREADS = 16

constexpr WriteFn< DatabaseMetadata >	write_database_metadata

constexpr ReadFn< DatabaseMetadata >	read_database_metadata

constexpr WriteFn< BulkJobMetadata >	write_bulk_job_metadata

constexpr ReadFn< BulkJobMetadata >	read_bulk_job_metadata

constexpr WriteFn< TableMetadata >	write_table_metadata

constexpr ReadFn< TableMetadata >	read_table_metadata

constexpr WriteFn< VideoMetadata >	write_video_metadata

constexpr ReadFn< VideoMetadata >	read_video_metadata

Detailed Description

Implementation Details.

Variable Documentation

const std::vector<std::string> scanner::internal::BUILTIN_OP_NAMES

Initial value:

= {
  SAMPLE_OP_NAME,
  SPACE_OP_NAME,
  SLICE_OP_NAME,
  UNSLICE_OP_NAME,
}

constexpr ReadFn<BulkJobMetadata> scanner::internal::read_bulk_job_metadata

Initial value:

=

read_db_proto<BulkJobMetadata>

constexpr ReadFn<DatabaseMetadata> scanner::internal::read_database_metadata

Initial value:

=

read_db_proto<DatabaseMetadata>

constexpr ReadFn<TableMetadata> scanner::internal::read_table_metadata

Initial value:

=

read_db_proto<TableMetadata>

constexpr ReadFn<VideoMetadata> scanner::internal::read_video_metadata

Initial value:

=

read_db_proto<VideoMetadata>

constexpr WriteFn<BulkJobMetadata> scanner::internal::write_bulk_job_metadata

Initial value:

=

write_db_proto<BulkJobMetadata>

constexpr WriteFn<DatabaseMetadata> scanner::internal::write_database_metadata

Initial value:

=

write_db_proto<DatabaseMetadata>

constexpr WriteFn<TableMetadata> scanner::internal::write_table_metadata

Initial value:

=

write_db_proto<TableMetadata>

constexpr WriteFn<VideoMetadata> scanner::internal::write_video_metadata

Initial value:

=

write_db_proto<VideoMetadata>

Classes

Typedefs

Enumerations

Functions

Variables

Detailed Description

Variable Documentation