Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for logging strings #178

Merged
merged 1 commit into from
Apr 23, 2024
Merged

Add support for logging strings #178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 65 additions & 11 deletions locustdb-client/src/client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
use locustdb_compression_utils::xor_float;
use locustdb_serialization::api::{Column, ColumnNameRequest, ColumnNameResponse, EncodingOpts, MultiQueryRequest, MultiQueryResponse};
use locustdb_serialization::api::{
AnyVal, Column, ColumnNameRequest, ColumnNameResponse, EncodingOpts, MultiQueryRequest,
MultiQueryResponse,
};
use locustdb_serialization::event_buffer::{ColumnBuffer, ColumnData, EventBuffer, TableBuffer};
use reqwest::header::CONTENT_TYPE;
use wasm_bindgen::prelude::*;
use std::sync::Once;
use wasm_bindgen::prelude::*;

#[wasm_bindgen]
pub struct Client {
Expand Down Expand Up @@ -50,7 +54,14 @@ impl Client {
Ok(serde_wasm_bindgen::to_value(&rsps.columns).unwrap())
}

pub async fn multi_query(&self, queries: Vec<String>, binary: bool, compress: bool, mantissa: u32, full_precision_cols: Vec<String>) -> Result<JsValue, JsValue> {
pub async fn multi_query(
&self,
queries: Vec<String>,
binary: bool,
compress: bool,
mantissa: u32,
full_precision_cols: Vec<String>,
) -> Result<JsValue, JsValue> {
let window = web_sys::window().expect("should have a window in this context");
let performance = window
.performance()
Expand All @@ -63,7 +74,10 @@ impl Client {
xor_float_compression: compress,
mantissa: Some(mantissa),
full_precision_cols: full_precision_cols.into_iter().collect(),
}) } else { None },
})
} else {
None
},
};
let request_start_ms = performance.now();
let response = self
Expand All @@ -82,11 +96,7 @@ impl Client {
let mut rsps = MultiQueryResponse::deserialize(&bytes).unwrap().responses;
rsps.iter_mut().for_each(|rsp| {
rsp.columns.iter_mut().for_each(|(key, col)| {
let compressed_bytes = if self.log_stats {
col.size_bytes()
} else {
0
};
let compressed_bytes = if self.log_stats { col.size_bytes() } else { 0 };
let coltype = match col {
Column::Float(_) => "float",
Column::Int(_) => "int",
Expand All @@ -96,7 +106,7 @@ impl Client {
Column::Xor(_) => "xor",
};
if let Column::Xor(compressed) = col {
*col = Column::Float(xor_float::double::decode(&compressed[..]).unwrap());
*col = Column::Float(xor_float::double::decode(&compressed[..]).unwrap());
};
if self.log_stats {
log::info!(
Expand All @@ -123,4 +133,48 @@ impl Client {
};
Ok(serde_wasm_bindgen::to_value(&rsps).unwrap())
}
}

pub async fn insert(
&self,
table: &str,
columns: Vec<String>,
values: Vec<JsValue>,
) -> Result<(), JsValue> {
let columns = columns
.into_iter()
.zip(values.iter())
.map(|(name, value)| {
let val = js_value_to_any_val(value.clone());
let mut buffer = ColumnBuffer {
data: ColumnData::default(),
};
buffer.push(val, 0);
(name, buffer)
})
.collect();
let payload = EventBuffer {
tables: [(table.to_string(), TableBuffer { len: 1, columns })]
.iter()
.cloned()
.collect(),
};
let body = payload.serialize();
self.client
.post(&format!("{}/insert_bin", self.url))
.body(body)
.send()
.await
.map_err(|e| JsValue::from_str(&format!("{}", e)))?;
Ok(())
}
}

fn js_value_to_any_val(value: JsValue) -> AnyVal {
if let Some(value) = value.as_f64() {
AnyVal::Float(value)
} else if let Some(value) = value.as_string() {
AnyVal::Str(value)
} else {
panic!("unsupported type")
}
}
5 changes: 5 additions & 0 deletions locustdb-serialization/schemas/wal_segment.capnp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,10 @@ struct Column {
}
i64 @4 :List(Int64);
string @5 :List(Text);
empty @6 :Void;
sparseI64 :group {
indices @7 :List(UInt64);
values @8 :List(Int64);
}
}
}
20 changes: 12 additions & 8 deletions locustdb-serialization/src/api.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,15 +226,19 @@ impl QueryResponse {
}
}
}
Column::Null(n) => column_builder
.reborrow()
.init_data()
.set_null(*n as u64),
Column::Xor(xs) => column_builder
.reborrow()
.init_data()
.set_xor_f64(&xs[..]),
Column::Null(n) => column_builder.reborrow().init_data().set_null(*n as u64),
Column::Xor(xs) => column_builder.reborrow().init_data().set_xor_f64(&xs[..]),
};
}
}
}

pub mod any_val_syntax {
pub fn vf64<F>(x: F) -> super::AnyVal
where
F: TryInto<f64>,
<F as TryInto<f64>>::Error: std::fmt::Debug,
{
super::AnyVal::Float(x.try_into().unwrap())
}
}
114 changes: 95 additions & 19 deletions locustdb-serialization/src/event_buffer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use std::collections::HashMap;

use crate::api::AnyVal;
use crate::wal_segment_capnp::{self, table_segment_list};

#[derive(Default, Clone, Debug)]
Expand All @@ -18,11 +19,14 @@ pub struct ColumnBuffer {
pub data: ColumnData,
}

#[derive(Clone, Debug)]
#[derive(Clone, Debug, Default)]
pub enum ColumnData {
#[default]
Empty,
Dense(Vec<f64>),
Sparse(Vec<(u64, f64)>),
I64(Vec<i64>),
SparseI64(Vec<(u64, i64)>),
String(Vec<String>),
}

Expand All @@ -32,7 +36,9 @@ impl ColumnData {
ColumnData::Dense(data) => data.len(),
ColumnData::Sparse(data) => data.len(),
ColumnData::I64(data) => data.len(),
ColumnData::SparseI64(data) => data.len(),
ColumnData::String(data) => data.len(),
ColumnData::Empty => 0,
}
}

Expand All @@ -42,43 +48,98 @@ impl ColumnData {
ColumnData::Dense(data) => data.is_empty(),
ColumnData::Sparse(data) => data.is_empty(),
ColumnData::I64(data) => data.is_empty(),
ColumnData::SparseI64(data) => data.is_empty(),
ColumnData::String(data) => data.is_empty(),
ColumnData::Empty => true,
}
}
}

impl ColumnBuffer {
pub fn push(&mut self, value: f64, len: u64) {
match &mut self.data {
ColumnData::Dense(data) => {
if data.len() as u64 == len {
pub fn push(&mut self, value: AnyVal, existing_len: u64) {
match (&mut self.data, value) {
(ColumnData::Empty, AnyVal::Float(value)) => {
if existing_len == 0 {
self.data = ColumnData::Dense(vec![value])
} else {
self.data = ColumnData::Sparse(vec![(existing_len, value)])
}
}
(ColumnData::Empty, AnyVal::Int(value)) => {
if existing_len == 0 {
self.data = ColumnData::I64(vec![value])
} else {
self.data = ColumnData::SparseI64(vec![(existing_len, value)])
}
}
(ColumnData::Empty, AnyVal::Str(value)) => {
assert!(
existing_len == 0,
"Sparse columns not currently supported for string"
);
self.data = ColumnData::String(vec![value])
}
(ColumnData::Dense(_), AnyVal::Int(int)) => {
self.push(AnyVal::Float(int as f64), existing_len)
}
(ColumnData::Dense(data), AnyVal::Float(value)) => {
if data.len() as u64 == existing_len {
data.push(value)
} else {
let mut sparse_data: Vec<(u64, f64)> = data
.drain(..)
.enumerate()
.map(|(i, v)| (i as u64, v))
.collect();
sparse_data.push((len, value));
sparse_data.push((existing_len, value));
self.data = ColumnData::Sparse(sparse_data);
}
}
ColumnData::Sparse(data) => data.push((len, value)),
_ => unimplemented!("Cannot push to non-f64 column"),
(ColumnData::Sparse(_), AnyVal::Int(value)) => {
self.push(AnyVal::Float(value as f64), existing_len)
}
(ColumnData::Sparse(data), AnyVal::Float(value)) => data.push((existing_len, value)),
(ColumnData::I64(data), AnyVal::Int(value)) => {
if data.len() as u64 == existing_len {
data.push(value)
} else {
let mut sparse_data: Vec<(u64, i64)> = data
.drain(..)
.enumerate()
.map(|(i, v)| (i as u64, v))
.collect();
sparse_data.push((existing_len, value));
self.data = ColumnData::SparseI64(sparse_data);
}
}
(ColumnData::I64(data), AnyVal::Float(value)) => {
self.data = ColumnData::Dense(data.iter().map(|v| *v as f64).collect());
self.push(AnyVal::Float(value), existing_len);
}
(ColumnData::SparseI64(data), AnyVal::Int(value)) => {
data.push((existing_len, value));
}
(ColumnData::SparseI64(data), AnyVal::Float(value)) => {
self.data = ColumnData::Sparse(data.iter().map(|(i, v)| (*i, *v as f64)).collect());
self.push(AnyVal::Float(value), existing_len);
}
(ColumnData::String(data), AnyVal::Str(value)) => {
assert!(
data.len() as u64 == existing_len,
"Sparse columns not currently supported for string"
);
data.push(value)
}
(column, data) => unimplemented!("Logging value {:?} to column {:?}", data, column),
}
}
}

impl Default for ColumnData {
fn default() -> Self {
ColumnData::Dense(Vec::new())
}
}

impl EventBuffer {
pub fn serialize(&self) -> Vec<u8> {
let mut builder = capnp::message::Builder::new_default();
let mut table_segment_list = builder.init_root::<wal_segment_capnp::table_segment_list::Builder>();
let mut table_segment_list =
builder.init_root::<wal_segment_capnp::table_segment_list::Builder>();
self.serialize_builder(&mut table_segment_list);
let mut buf = Vec::new();
capnp::serialize_packed::write_message(&mut buf, &builder).unwrap();
Expand Down Expand Up @@ -117,6 +178,16 @@ impl EventBuffer {
ColumnData::String(strings) => {
column_builder.get_data().set_string(&strings[..]).unwrap();
}
ColumnData::Empty => {
column_builder.get_data().set_empty(());
}
ColumnData::SparseI64(sparse) => {
let mut sparse_builder = column_builder.get_data().init_sparse_i64();
assert!(sparse.len() < std::u32::MAX as usize);
let (indices, values): (Vec<_>, Vec<_>) = sparse.iter().cloned().unzip();
sparse_builder.reborrow().set_indices(&indices[..]).unwrap();
sparse_builder.reborrow().set_values(&values[..]).unwrap();
}
}
}
}
Expand All @@ -125,7 +196,8 @@ impl EventBuffer {
pub fn deserialize(data: &[u8]) -> capnp::Result<Self> {
let message_reader =
capnp::serialize_packed::read_message(data, capnp::message::ReaderOptions::new())?;
let table_segment_list = message_reader.get_root::<wal_segment_capnp::table_segment_list::Reader>()?;
let table_segment_list =
message_reader.get_root::<wal_segment_capnp::table_segment_list::Reader>()?;
let data = EventBuffer::deserialize_reader(table_segment_list)?;
Ok(EventBuffer {
tables: data.tables,
Expand Down Expand Up @@ -157,13 +229,17 @@ impl EventBuffer {
}
ColumnData::String(strings)
}
Which::Empty(()) => ColumnData::Empty,
Which::SparseI64(sparse) => {
let indices = sparse.get_indices()?;
let values = sparse.get_values()?;
ColumnData::SparseI64(indices.iter().zip(values.iter()).collect())
}
};
columns.insert(colname, ColumnBuffer { data });
}
tables.insert(name, TableBuffer { len, columns });
}
Ok(EventBuffer {
tables,
})
Ok(EventBuffer { tables })
}
}
Loading
Loading