sig
val invoke : ('a -> 'b) Mapred_rfun.rfun -> 'a -> 'b
type format =
[ `Auto_input | `Fixed_size of int | `Line_structured | `Var_size ]
module Place :
sig
type 'a t
type 'a codec =
('a -> string) Mapred_rfun.rfun *
(string -> int -> int -> 'a) Mapred_rfun.rfun
type ext_location =
[ `Deep_dir of string | `File of string | `Flat_dir of string ]
type location =
[ `Deep_dir of string
| `File of string
| `Flat_dir of string
| `Notebook ]
val create :
?prefix:string ->
?repl:int ->
Mapred_fs.filesystem ->
'a Mapred_toolkit.Place.codec ->
Mapred_toolkit.format ->
Mapred_io.record_config ->
Mapred_toolkit.Place.ext_location -> 'a Mapred_toolkit.Place.t
val from :
?prefix:string ->
?repl:int ->
Mapred_fs.filesystem ->
'a Mapred_toolkit.Place.codec ->
Mapred_toolkit.format ->
Mapred_io.record_config ->
Mapred_toolkit.Place.ext_location -> 'a Mapred_toolkit.Place.t
val notebook_place : unit -> 'a Mapred_toolkit.Place.t
val location :
'a Mapred_toolkit.Place.t -> Mapred_toolkit.Place.location
val get_codec :
'a Mapred_toolkit.Place.t -> 'a Mapred_toolkit.Place.codec
val get_format : 'a Mapred_toolkit.Place.t -> Mapred_toolkit.format
val get_fs : 'a Mapred_toolkit.Place.t -> Mapred_fs.filesystem
val get_rc : 'a Mapred_toolkit.Place.t -> Mapred_io.record_config
val is_file_place : 'a Mapred_toolkit.Place.t -> bool
val files : 'a Mapred_toolkit.Place.t -> string list
val create_file :
?filename:string -> 'a Mapred_toolkit.Place.t -> string
val clear : 'a Mapred_toolkit.Place.t -> unit
end
module Store :
sig
type 'a store
val notebook : unit -> 'a Mapred_toolkit.Store.store
val place : 'a Mapred_toolkit.Store.store -> 'a Mapred_toolkit.Place.t
val read_place :
'a Mapred_toolkit.Place.t -> 'a Mapred_toolkit.Store.store list
val write_place :
?filename:string ->
'a Mapred_toolkit.Place.t -> 'a Mapred_toolkit.Store.store
val length : 'a Mapred_toolkit.Store.store -> int
val length64 : 'a Mapred_toolkit.Store.store -> int64
val read :
'a Mapred_toolkit.Store.store -> 'a Mapred_toolkit.Store.store
val file_name : 'a Mapred_toolkit.Store.store -> string
val flush : 'a Mapred_toolkit.Store.store -> unit
val close : 'a Mapred_toolkit.Store.store -> unit
end
module Seq :
sig
type ('a, 'b) seq
exception End_of_sequence
val read :
'a Mapred_toolkit.Store.store -> ('a, [ `R ]) Mapred_toolkit.Seq.seq
val extend :
'a Mapred_toolkit.Store.store -> ('a, [ `W ]) Mapred_toolkit.Seq.seq
val reread :
('a, 'b) Mapred_toolkit.Seq.seq ->
('a, [ `R ]) Mapred_toolkit.Seq.seq
val store :
('a, 'b) Mapred_toolkit.Seq.seq -> 'a Mapred_toolkit.Store.store
val notebook : unit -> ('a, [ `R | `W ]) Mapred_toolkit.Seq.seq
val is_empty : ('a, [> `R ]) Mapred_toolkit.Seq.seq -> bool
val hd : ('a, [> `R ]) Mapred_toolkit.Seq.seq -> 'a
val tl :
('a, [> `R ] as 'm) Mapred_toolkit.Seq.seq ->
('a, 'm) Mapred_toolkit.Seq.seq
val add :
'a ->
('a, [> `W ] as 'm) Mapred_toolkit.Seq.seq ->
('a, 'm) Mapred_toolkit.Seq.seq
val addl :
'a list ->
('a, [> `W ] as 'm) Mapred_toolkit.Seq.seq ->
('a, 'm) Mapred_toolkit.Seq.seq
val append :
('a, [> `W ] as 'm) Mapred_toolkit.Seq.seq ->
('a, [> `R ]) Mapred_toolkit.Seq.seq ->
('a, 'm) Mapred_toolkit.Seq.seq
val flush : ('a, 'b) Mapred_toolkit.Seq.seq -> unit
val length : ('a, 'b) Mapred_toolkit.Seq.seq -> int
val length64 : ('a, 'b) Mapred_toolkit.Seq.seq -> int64
val hdl : int -> ('a, [> `R ]) Mapred_toolkit.Seq.seq -> 'a list
val iter : ('a -> unit) -> ('a, [> `R ]) Mapred_toolkit.Seq.seq -> unit
val fold :
('a -> 'b -> 'a) -> 'a -> ('b, [> `R ]) Mapred_toolkit.Seq.seq -> 'a
val map :
('a -> 'b) ->
('a, [> `R ]) Mapred_toolkit.Seq.seq ->
('b, [> `W ] as 'm) Mapred_toolkit.Seq.seq ->
('b, 'm) Mapred_toolkit.Seq.seq
val mapl :
('a -> 'b list) ->
('a, [> `R ]) Mapred_toolkit.Seq.seq ->
('b, [> `W ] as 'm) Mapred_toolkit.Seq.seq ->
('b, 'm) Mapred_toolkit.Seq.seq
val sort :
hash:('a -> int) ->
cmp:('a -> 'a -> int) ->
('a, [> `R ]) Mapred_toolkit.Seq.seq ->
('a, [> `W ] as 'm) Mapred_toolkit.Seq.seq ->
('a, 'm) Mapred_toolkit.Seq.seq
val mapl_sort_fold :
mapl:('a -> 'b list) ->
hash:('b -> int) ->
cmp:('b -> 'b -> int) ->
initfold:(int -> 'c) ->
fold:('c -> 'b -> 'c * 'd list) ->
?finfold:('c -> 'd list) ->
partition_of:('b -> int) ->
partitions:int ->
'a Mapred_toolkit.Place.t ->
'd Mapred_toolkit.Place.t -> ('d, [ `W ]) Mapred_toolkit.Seq.seq list
end
module DSeq :
sig
type config
val create_config :
?name:string ->
?task_files:string list ->
?bigblock_size:int ->
?map_tasks:int ->
?merge_limit:int ->
?split_limit:int ->
?partitions:int ->
?enhanced_mapping:int ->
?phases:Mapred_def.phases ->
?report:bool ->
?report_to:Netchannels.out_obj_channel ->
?keep_temp_files:bool ->
Mapred_def.mapred_env -> Mapred_toolkit.DSeq.config
val get_rc : Mapred_toolkit.DSeq.config -> Mapred_io.record_config
type 'a result
val get_result : 'a Mapred_toolkit.DSeq.result -> 'a
val stats : 'a Mapred_toolkit.DSeq.result -> Mapred_stats.stats
val job_id : Mapred_toolkit.DSeq.config -> string
class type mapred_info =
object
method mapred_env : Mapred_def.mapred_env
method mapred_job_config : Mapred_def.mapred_job_config
end
val mapl :
(Mapred_toolkit.DSeq.mapred_info -> 'a -> 'b list) Mapred_rfun.rfun ->
'a Mapred_toolkit.Place.t ->
'b Mapred_toolkit.Place.t ->
Mapred_toolkit.DSeq.config ->
('b, [ `W ]) Mapred_toolkit.Seq.seq list Mapred_toolkit.DSeq.result
val mapl_sort_fold :
mapl:(Mapred_toolkit.DSeq.mapred_info -> 'a -> 'b list)
Mapred_rfun.rfun ->
hash:(Mapred_toolkit.DSeq.mapred_info -> 'b -> int) Mapred_rfun.rfun ->
cmp:(Mapred_toolkit.DSeq.mapred_info -> 'b -> 'b -> int)
Mapred_rfun.rfun ->
initfold:(Mapred_toolkit.DSeq.mapred_info -> int -> 'c)
Mapred_rfun.rfun ->
fold:(Mapred_toolkit.DSeq.mapred_info -> 'c -> 'b -> 'c * 'd list)
Mapred_rfun.rfun ->
?finfold:(Mapred_toolkit.DSeq.mapred_info -> 'c -> 'd list)
Mapred_rfun.rfun ->
partition_of:(Mapred_toolkit.DSeq.mapred_info -> 'b -> int)
Mapred_rfun.rfun ->
?initcombine:(Mapred_toolkit.DSeq.mapred_info -> 'e) Mapred_rfun.rfun ->
?combine:(Mapred_toolkit.DSeq.mapred_info -> 'e -> 'b -> 'e * 'b list)
Mapred_rfun.rfun ->
?fincombine:(Mapred_toolkit.DSeq.mapred_info -> 'e -> 'b list)
Mapred_rfun.rfun ->
'a Mapred_toolkit.Place.t ->
'd Mapred_toolkit.Place.t ->
Mapred_toolkit.DSeq.config ->
'b Mapred_toolkit.Place.codec ->
('d, [ `W ]) Mapred_toolkit.Seq.seq list Mapred_toolkit.DSeq.result
end
val toolkit_job : Mapred_def.mapred_env -> Mapred_def.mapred_job
class toolkit_job : Mapred_def.mapred_env -> Mapred_def.mapred_job
end
