Skip to content

Commit

Permalink
Update vendored sources to duckdb/duckdb@84c7b8e
Browse files Browse the repository at this point in the history
  • Loading branch information
@Tmonster
Tmonster committed Sep 8, 2023
1 parent 976ce8d commit 0515a7d
Show file tree
Hide file tree
Showing 1,022 changed files with 40,314 additions and 30,983 deletions.
62 changes: 37 additions & 25 deletions src/duckdb/extension/parquet/column_reader.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,25 @@
#include "column_reader.hpp"
#include "parquet_timestamp.hpp"
#include "utf8proc_wrapper.hpp"
#include "parquet_reader.hpp"

#include "boolean_column_reader.hpp"
#include "cast_column_reader.hpp"
#include "row_number_column_reader.hpp"
#include "callback_column_reader.hpp"
#include "parquet_decimal_utils.hpp"
#include "cast_column_reader.hpp"
#include "duckdb.hpp"
#include "list_column_reader.hpp"
#include "miniz_wrapper.hpp"
#include "parquet_decimal_utils.hpp"
#include "parquet_reader.hpp"
#include "parquet_timestamp.hpp"
#include "row_number_column_reader.hpp"
#include "snappy.h"
#include "string_column_reader.hpp"
#include "struct_column_reader.hpp"
#include "templated_column_reader.hpp"

#include "snappy.h"
#include "miniz_wrapper.hpp"
#include "utf8proc_wrapper.hpp"
#include "zstd.h"
#include <iostream>

#include "duckdb.hpp"
#ifndef DUCKDB_AMALGAMATION
#include "duckdb/common/types/blob.hpp"
#include "duckdb/common/types/bit.hpp"
#include "duckdb/common/types/blob.hpp"
#include "duckdb/common/types/chunk_collection.hpp"
#endif

Expand Down Expand Up @@ -509,13 +507,13 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr
// TODO keep this in the state
auto read_buf = make_shared<ResizeableBuffer>();

switch (type.InternalType()) {
case PhysicalType::INT32:
switch (schema.type) {
case duckdb_parquet::format::Type::INT32:
read_buf->resize(reader.allocator, sizeof(int32_t) * (read_now - null_count));
dbp_decoder->GetBatch<int32_t>(read_buf->ptr, read_now - null_count);

break;
case PhysicalType::INT64:
case duckdb_parquet::format::Type::INT64:
read_buf->resize(reader.allocator, sizeof(int64_t) * (read_now - null_count));
dbp_decoder->GetBatch<int64_t>(read_buf->ptr, read_now - null_count);
break;
Expand Down Expand Up @@ -614,7 +612,7 @@ uint32_t StringColumnReader::VerifyString(const char *str_data, uint32_t str_len

void StringColumnReader::Dictionary(shared_ptr<ResizeableBuffer> data, idx_t num_entries) {
dict = std::move(data);
dict_strings = duckdb::unique_ptr<string_t[]>(new string_t[num_entries]);
dict_strings = unique_ptr<string_t[]>(new string_t[num_entries]);
for (idx_t dict_idx = 0; dict_idx < num_entries; dict_idx++) {
uint32_t str_len;
if (fixed_width_string_length == 0) {
Expand Down Expand Up @@ -873,7 +871,7 @@ idx_t ListColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data

ListColumnReader::ListColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p,
idx_t schema_idx_p, idx_t max_define_p, idx_t max_repeat_p,
duckdb::unique_ptr<ColumnReader> child_column_reader_p)
unique_ptr<ColumnReader> child_column_reader_p)
: ColumnReader(reader, std::move(type_p), schema_p, schema_idx_p, max_define_p, max_repeat_p),
child_column_reader(std::move(child_column_reader_p)),
read_cache(reader.allocator, ListType::GetChildType(Type())), read_vector(read_cache), overflow_child_count(0) {
Expand All @@ -889,8 +887,8 @@ ListColumnReader::ListColumnReader(ParquetReader &reader, LogicalType type_p, co
void ListColumnReader::ApplyPendingSkips(idx_t num_values) {
pending_skips -= num_values;

auto define_out = duckdb::unique_ptr<uint8_t[]>(new uint8_t[num_values]);
auto repeat_out = duckdb::unique_ptr<uint8_t[]>(new uint8_t[num_values]);
auto define_out = unique_ptr<uint8_t[]>(new uint8_t[num_values]);
auto repeat_out = unique_ptr<uint8_t[]>(new uint8_t[num_values]);

idx_t remaining = num_values;
idx_t read = 0;
Expand Down Expand Up @@ -953,7 +951,7 @@ idx_t RowNumberColumnReader::Read(uint64_t num_values, parquet_filter_t &filter,
//===--------------------------------------------------------------------===//
// Cast Column Reader
//===--------------------------------------------------------------------===//
CastColumnReader::CastColumnReader(duckdb::unique_ptr<ColumnReader> child_reader_p, LogicalType target_type_p)
CastColumnReader::CastColumnReader(unique_ptr<ColumnReader> child_reader_p, LogicalType target_type_p)
: ColumnReader(child_reader_p->Reader(), std::move(target_type_p), child_reader_p->Schema(),
child_reader_p->FileIdx(), child_reader_p->MaxDefine(), child_reader_p->MaxRepeat()),
child_reader(std::move(child_reader_p)) {
Expand Down Expand Up @@ -1005,7 +1003,7 @@ idx_t CastColumnReader::GroupRowsAvailable() {
//===--------------------------------------------------------------------===//
StructColumnReader::StructColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p,
idx_t schema_idx_p, idx_t max_define_p, idx_t max_repeat_p,
vector<duckdb::unique_ptr<ColumnReader>> child_readers_p)
vector<unique_ptr<ColumnReader>> child_readers_p)
: ColumnReader(reader, std::move(type_p), schema_p, schema_idx_p, max_define_p, max_repeat_p),
child_readers(std::move(child_readers_p)) {
D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
Expand Down Expand Up @@ -1155,9 +1153,9 @@ class DecimalColumnReader
};

template <bool FIXED_LENGTH>
static duckdb::unique_ptr<ColumnReader> CreateDecimalReaderInternal(ParquetReader &reader, const LogicalType &type_p,
const SchemaElement &schema_p, idx_t file_idx_p,
idx_t max_define, idx_t max_repeat) {
static unique_ptr<ColumnReader> CreateDecimalReaderInternal(ParquetReader &reader, const LogicalType &type_p,
const SchemaElement &schema_p, idx_t file_idx_p,
idx_t max_define, idx_t max_repeat) {
switch (type_p.InternalType()) {
case PhysicalType::INT16:
return make_uniq<DecimalColumnReader<int16_t, FIXED_LENGTH>>(reader, type_p, schema_p, file_idx_p, max_define,
Expand Down Expand Up @@ -1391,7 +1389,6 @@ unique_ptr<ColumnReader> ColumnReader::CreateReader(ParquetReader &reader, const
return make_uniq<CallbackColumnReader<int32_t, date_t, ParquetIntToDate>>(reader, type_p, schema_p, file_idx_p,
max_define, max_repeat);
case LogicalTypeId::TIME:
case LogicalTypeId::TIME_TZ:
if (schema_p.__isset.logicalType && schema_p.logicalType.__isset.TIME) {
if (schema_p.logicalType.TIME.unit.__isset.MILLIS) {
return make_uniq<CallbackColumnReader<int32_t, dtime_t, ParquetIntToTimeMs>>(
Expand All @@ -1415,6 +1412,21 @@ unique_ptr<ColumnReader> ColumnReader::CreateReader(ParquetReader &reader, const
break;
}
}
case LogicalTypeId::TIME_TZ:
if (schema_p.__isset.logicalType && schema_p.logicalType.__isset.TIME) {
if (schema_p.logicalType.TIME.unit.__isset.MICROS) {
return make_uniq<CallbackColumnReader<int64_t, dtime_tz_t, ParquetIntToTimeTZ>>(
reader, type_p, schema_p, file_idx_p, max_define, max_repeat);
}
} else if (schema_p.__isset.converted_type) {
switch (schema_p.converted_type) {
case ConvertedType::TIME_MICROS:
return make_uniq<CallbackColumnReader<int64_t, dtime_tz_t, ParquetIntToTimeTZ>>(
reader, type_p, schema_p, file_idx_p, max_define, max_repeat);
default:
break;
}
}
case LogicalTypeId::BLOB:
case LogicalTypeId::VARCHAR:
return make_uniq<StringColumnReader>(reader, type_p, schema_p, file_idx_p, max_define, max_repeat);
Expand Down
Loading

0 comments on commit 0515a7d

Please sign in to comment.