sig
class type record_config =
object
method bigblock_size : int
method mr_buffer_size : int
method mr_buffer_size_tight : int
end
class type record_reader =
object
method abort : unit -> unit
method close_in : unit -> unit
method filesystem : Mapred_fs.filesystem
method input_record : unit -> string
method input_records : string Queue.t -> unit
method peek_record : unit -> string
method pos_in : int
method record_config : Mapred_io.record_config
method stats : Mapred_stats.stats
method to_any_e :
(Netsys_mem.memory -> int -> int -> unit Uq_engines.engine) ->
Unixqueue.event_system -> unit Uq_engines.engine
method to_dev_e :
Uq_io.out_device -> Unixqueue.event_system -> unit Uq_engines.engine
method to_fd_e :
Unix.file_descr -> Unixqueue.event_system -> unit Uq_engines.engine
end
class type record_writer =
object
method abort : unit -> unit
method close_out : unit -> unit
method filesystem : Mapred_fs.filesystem
method flush : unit -> unit
method from_dev_e :
string Queue.t ->
Uq_io.in_bdevice ->
int64 option ->
int option -> Unixqueue.event_system -> bool Uq_engines.engine
method from_fd_e :
Unix.file_descr -> Unixqueue.event_system -> unit Uq_engines.engine
method output_record : string -> unit
method output_records : string Queue.t -> unit
method record_config : Mapred_io.record_config
method stats : Mapred_stats.stats
end
type read_flag = [ `Bof_block of int64 | `Eof_pos of int64 ]
class type record_reader_factory =
object
method read_file :
Mapred_io.read_flag list ->
Mapred_fs.filesystem ->
Plasma_shm.shm_manager ->
Mapred_io.record_config ->
string -> int64 -> int64 -> Mapred_io.record_reader
end
class type record_writer_factory =
object
method write_file :
Mapred_fs.filesystem ->
Plasma_shm.shm_manager ->
Mapred_io.record_config -> string -> Mapred_io.record_writer
method write_suffix : string
end
class type record_rw_factory =
object
method read_file :
read_flag list ->
Mapred_fs.filesystem ->
Plasma_shm.shm_manager ->
record_config -> string -> int64 -> int64 -> record_reader
method write_file :
Mapred_fs.filesystem ->
Plasma_shm.shm_manager -> record_config -> string -> record_writer
method write_suffix : string
end
val line_structured_format : unit -> Mapred_io.record_rw_factory
val fixed_size_format : int -> Mapred_io.record_rw_factory
val var_size_format : unit -> Mapred_io.record_rw_factory
val auto_input_format : unit -> Mapred_io.record_rw_factory
val bigblock_size : Mapred_fs.filesystem -> string -> int -> int
type sync_readers = (unit -> Mapred_io.record_reader) list
type async_readers =
(unit -> Mapred_io.record_reader Uq_engines.engine) list
val read_multiple :
Mapred_fs.filesystem ->
Mapred_io.record_config ->
readers:[ `Async of Mapred_io.async_readers
| `Sync of Mapred_io.sync_readers ] ->
unit -> Mapred_io.record_reader
val write_multiple :
Mapred_fs.filesystem ->
Plasma_shm.shm_manager ->
Mapred_io.record_config ->
string ->
int64 ->
int ->
create_sync:(string -> int -> string) ->
create_async:(string -> int -> string Uq_engines.engine) option ->
Mapred_io.record_writer_factory -> Mapred_io.record_writer
val divide : Mapred_io.record_config -> int -> Mapred_io.record_config
val file_exists : Mapred_fs.filesystem -> string -> bool
val is_directory : Mapred_fs.filesystem -> string -> bool
val create_file :
?repl:int ->
?pref_nodes:string list -> Mapred_fs.filesystem -> string -> unit
val create_dir : Mapred_fs.filesystem -> string -> unit
val delete_file : Mapred_fs.filesystem -> string -> unit
val delete_rec : Mapred_fs.filesystem -> string -> unit
val file_blocks : Mapred_fs.filesystem -> string -> int64
type fileinfo = [ `Directory | `Other | `Regular of int64 ]
val scan_file : Mapred_fs.filesystem -> string -> Mapred_io.fileinfo
val scan_dir :
Mapred_fs.filesystem ->
string -> bool -> (string * Mapred_io.fileinfo) list
class vs_write_file :
?chunklen:int ->
Mapred_fs.filesystem ->
Plasma_shm.shm_manager ->
Mapred_io.record_config -> string -> record_writer
val more_logging : bool Pervasives.ref
end
