Skip to content

Commit 449aed5

Browse files
AbelPauAbelPau
committed
Merge remote-tracking branch 'upstream/master'
2 parents 660d351 + 0332c83 commit 449aed5

File tree

12 files changed

+293
-7
lines changed

12 files changed

+293
-7
lines changed

‎autotest/ogr/ogr_gpkg.py

+46
Original file line numberDiff line numberDiff line change
@@ -10540,3 +10540,49 @@ def test_gpkg_create_more_than_2000_fields(tmp_vsimem):
1054010540
with pytest.raises(Exception, match="Limit of 2000 columns reached"):
1054110541
lyr.CreateField(ogr.FieldDefn("foo"))
1054210542
assert lyr.GetLayerDefn().GetFieldCount() == 2000 - 2
10543+
10544+
10545+
###############################################################################
10546+
# Test that secure_delete is turned on
10547+
10548+
10549+
@gdaltest.enable_exceptions()
10550+
def test_gpkg_secure_delete(tmp_vsimem):
10551+
10552+
filename = str(tmp_vsimem / "secure_delete.gpkg")
10553+
with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds:
10554+
10555+
with ds.ExecuteSQL("PRAGMA secure_delete") as sql_lyr:
10556+
f = sql_lyr.GetNextFeature()
10557+
assert f.GetField(0) == 1
10558+
10559+
lyr = ds.CreateLayer("test")
10560+
lyr.CreateField(ogr.FieldDefn("foo"))
10561+
f = ogr.Feature(lyr.GetLayerDefn())
10562+
f["foo"] = "very_secret"
10563+
lyr.CreateFeature(f)
10564+
10565+
f = gdal.VSIFOpenL(filename, "rb")
10566+
data = gdal.VSIFReadL(1, 100000, f)
10567+
gdal.VSIFCloseL(f)
10568+
assert b"very_secret" in data
10569+
10570+
with ogr.Open(filename, update=1) as ds:
10571+
10572+
with ds.ExecuteSQL("PRAGMA secure_delete") as sql_lyr:
10573+
f = sql_lyr.GetNextFeature()
10574+
assert f.GetField(0) == 1
10575+
10576+
lyr = ds.GetLayer(0)
10577+
lyr.DeleteFeature(1)
10578+
10579+
f = gdal.VSIFOpenL(filename, "rb")
10580+
data = gdal.VSIFReadL(1, 100000, f)
10581+
gdal.VSIFCloseL(f)
10582+
assert b"very_secret" not in data
10583+
10584+
with gdaltest.config_option("OGR_SQLITE_PRAGMA", "secure_delete=0"):
10585+
with ogr.Open(filename, update=1) as ds:
10586+
with ds.ExecuteSQL("PRAGMA secure_delete") as sql_lyr:
10587+
f = sql_lyr.GetNextFeature()
10588+
assert f.GetField(0) == 0

‎autotest/osr/osr_basic.py

+10
Original file line numberDiff line numberDiff line change
@@ -2478,3 +2478,13 @@ def test_osr_basic_export_wkt_utm_south():
24782478

24792479
j = json.loads(srs.ExportToPROJJSON())
24802480
assert j["conversion"]["id"]["code"] == 16101
2481+
2482+
2483+
###############################################################################
2484+
2485+
2486+
def test_osr_basic_GetAuthorityListFromDatabase():
2487+
2488+
ret = osr.GetAuthorityListFromDatabase()
2489+
assert "EPSG" in ret
2490+
assert "PROJ" in ret

‎doc/source/drivers/vector/gpkg.rst

+12
Original file line numberDiff line numberDiff line change
@@ -809,6 +809,18 @@ Examples
809809
-sql "SELECT poly.id, other.foo FROM poly JOIN other_schema.other USING (id)" \
810810
-oo PRELUDE_STATEMENTS="ATTACH DATABASE 'other.gpkg' AS other_schema"
811811

812+
Secure deletion
813+
---------------
814+
815+
Depending on how SQLite3 is built, `secure deletion <https://www.sqlite.org/pragma.html#pragma_secure_delete>`__
816+
might or might not be enabled.
817+
Starting with GDAL 3.10, secure deletion is always enabled, unless
818+
``SECURE_DELETE`` is specified through the :config:`OGR_SQLITE_PRAGMA`
819+
configuration option.
820+
Note that secure deletion does not recover potential lost space, so running
821+
a `VACUUM <https://sqlite.org/lang_vacuum.html>`__ query is recommended to fully
822+
optimized a database that has been subject to updates or deletions.
823+
812824
See Also
813825
--------
814826

‎doc/source/drivers/vector/sqlite.rst

+11
Original file line numberDiff line numberDiff line change
@@ -556,6 +556,17 @@ and optimize it.
556556

557557
ogrinfo db.sqlite -sql "VACUUM"
558558

559+
Secure deletion
560+
---------------
561+
562+
Depending on how SQLite3 is built, `secure deletion <https://www.sqlite.org/pragma.html#pragma_secure_delete>`__
563+
might or might not be enabled.
564+
Starting with GDAL 3.10, secure deletion is always enabled, unless
565+
``SECURE_DELETE`` is specified through the :config:`OGR_SQLITE_PRAGMA`
566+
configuration option.
567+
Note that secure deletion does not recover potential lost space, so running
568+
a `VACUUM <https://sqlite.org/lang_vacuum.html>`__ query is recommended to fully
569+
optimized a database that has been subject to updates or deletions.
559570

560571
Example
561572
-------

‎ogr/ogr_srs_api.h

+2
Original file line numberDiff line numberDiff line change
@@ -1005,6 +1005,8 @@ OSRGetCRSInfoListFromDatabase(const char *pszAuthName,
10051005

10061006
void CPL_DLL OSRDestroyCRSInfoList(OSRCRSInfo **list);
10071007

1008+
char CPL_DLL **OSRGetAuthorityListFromDatabase(void);
1009+
10081010
/* -------------------------------------------------------------------- */
10091011
/* OGRCoordinateTransform C API. */
10101012
/* -------------------------------------------------------------------- */

‎ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp

+138
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,10 @@ inline bool OGRArrowLayer::IsHandledListOrMapType(
228228
itemTypeId == arrow::Type::HALF_FLOAT ||
229229
itemTypeId == arrow::Type::FLOAT ||
230230
itemTypeId == arrow::Type::DOUBLE ||
231+
#if ARROW_VERSION_MAJOR >= 18
232+
itemTypeId == arrow::Type::DECIMAL32 ||
233+
itemTypeId == arrow::Type::DECIMAL64 ||
234+
#endif
231235
itemTypeId == arrow::Type::DECIMAL128 ||
232236
itemTypeId == arrow::Type::DECIMAL256 ||
233237
itemTypeId == arrow::Type::STRING ||
@@ -422,6 +426,10 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR(
422426
// nanosecond accuracy
423427
break;
424428

429+
#if ARROW_VERSION_MAJOR >= 18
430+
case arrow::Type::DECIMAL32:
431+
case arrow::Type::DECIMAL64:
432+
#endif
425433
case arrow::Type::DECIMAL128:
426434
case arrow::Type::DECIMAL256:
427435
{
@@ -468,6 +476,10 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR(
468476
eSubType = OFSTFloat32;
469477
break;
470478
case arrow::Type::DOUBLE:
479+
#if ARROW_VERSION_MAJOR >= 18
480+
case arrow::Type::DECIMAL32:
481+
case arrow::Type::DECIMAL64:
482+
#endif
471483
case arrow::Type::DECIMAL128:
472484
case arrow::Type::DECIMAL256:
473485
eType = OFTRealList;
@@ -1290,6 +1302,23 @@ static void AddToArray(CPLJSONArray &oArray, const arrow::Array *array,
12901302
static_cast<const arrow::DoubleArray *>(array)->Value(nIdx));
12911303
break;
12921304
}
1305+
1306+
#if ARROW_VERSION_MAJOR >= 18
1307+
case arrow::Type::DECIMAL32:
1308+
{
1309+
oArray.Add(CPLAtof(static_cast<const arrow::Decimal32Array *>(array)
1310+
->FormatValue(nIdx)
1311+
.c_str()));
1312+
break;
1313+
}
1314+
case arrow::Type::DECIMAL64:
1315+
{
1316+
oArray.Add(CPLAtof(static_cast<const arrow::Decimal64Array *>(array)
1317+
->FormatValue(nIdx)
1318+
.c_str()));
1319+
break;
1320+
}
1321+
#endif
12931322
case arrow::Type::DECIMAL128:
12941323
{
12951324
oArray.Add(
@@ -1470,6 +1499,25 @@ static void AddToDict(CPLJSONObject &oDict, const std::string &osKey,
14701499
static_cast<const arrow::DoubleArray *>(array)->Value(nIdx));
14711500
break;
14721501
}
1502+
1503+
#if ARROW_VERSION_MAJOR >= 18
1504+
case arrow::Type::DECIMAL32:
1505+
{
1506+
oDict.Add(osKey,
1507+
CPLAtof(static_cast<const arrow::Decimal32Array *>(array)
1508+
->FormatValue(nIdx)
1509+
.c_str()));
1510+
break;
1511+
}
1512+
case arrow::Type::DECIMAL64:
1513+
{
1514+
oDict.Add(osKey,
1515+
CPLAtof(static_cast<const arrow::Decimal64Array *>(array)
1516+
->FormatValue(nIdx)
1517+
.c_str()));
1518+
break;
1519+
}
1520+
#endif
14731521
case arrow::Type::DECIMAL128:
14741522
{
14751523
oDict.Add(osKey,
@@ -1757,6 +1805,48 @@ static void ReadList(OGRFeature *poFeature, int i, int64_t nIdxInArray,
17571805
break;
17581806
}
17591807

1808+
#if ARROW_VERSION_MAJOR >= 18
1809+
case arrow::Type::DECIMAL32:
1810+
{
1811+
const auto values = std::static_pointer_cast<arrow::Decimal32Array>(
1812+
array->values());
1813+
const auto nIdxStart = array->value_offset(nIdxInArray);
1814+
const int nCount = array->value_length(nIdxInArray);
1815+
std::vector<double> aValues;
1816+
aValues.reserve(nCount);
1817+
for (int k = 0; k < nCount; k++)
1818+
{
1819+
if (values->IsNull(nIdxStart + k))
1820+
aValues.push_back(std::numeric_limits<double>::quiet_NaN());
1821+
else
1822+
aValues.push_back(
1823+
CPLAtof(values->FormatValue(nIdxStart + k).c_str()));
1824+
}
1825+
poFeature->SetField(i, nCount, aValues.data());
1826+
break;
1827+
}
1828+
1829+
case arrow::Type::DECIMAL64:
1830+
{
1831+
const auto values = std::static_pointer_cast<arrow::Decimal64Array>(
1832+
array->values());
1833+
const auto nIdxStart = array->value_offset(nIdxInArray);
1834+
const int nCount = array->value_length(nIdxInArray);
1835+
std::vector<double> aValues;
1836+
aValues.reserve(nCount);
1837+
for (int k = 0; k < nCount; k++)
1838+
{
1839+
if (values->IsNull(nIdxStart + k))
1840+
aValues.push_back(std::numeric_limits<double>::quiet_NaN());
1841+
else
1842+
aValues.push_back(
1843+
CPLAtof(values->FormatValue(nIdxStart + k).c_str()));
1844+
}
1845+
poFeature->SetField(i, nCount, aValues.data());
1846+
break;
1847+
}
1848+
#endif
1849+
17601850
case arrow::Type::DECIMAL128:
17611851
{
17621852
const auto values =
@@ -2406,6 +2496,26 @@ inline OGRFeature *OGRArrowLayer::ReadFeature(
24062496
break;
24072497
}
24082498

2499+
#if ARROW_VERSION_MAJOR >= 18
2500+
case arrow::Type::DECIMAL32:
2501+
{
2502+
const auto castArray =
2503+
static_cast<const arrow::Decimal32Array *>(array);
2504+
poFeature->SetField(
2505+
i, CPLAtof(castArray->FormatValue(nIdxInBatch).c_str()));
2506+
break;
2507+
}
2508+
2509+
case arrow::Type::DECIMAL64:
2510+
{
2511+
const auto castArray =
2512+
static_cast<const arrow::Decimal64Array *>(array);
2513+
poFeature->SetField(
2514+
i, CPLAtof(castArray->FormatValue(nIdxInBatch).c_str()));
2515+
break;
2516+
}
2517+
#endif
2518+
24092519
case arrow::Type::DECIMAL128:
24102520
{
24112521
const auto castArray =
@@ -3900,6 +4010,34 @@ inline bool OGRArrowLayer::SkipToNextFeatureDueToAttributeFilter() const
39004010
break;
39014011
}
39024012

4013+
#if ARROW_VERSION_MAJOR >= 18
4014+
case arrow::Type::DECIMAL32:
4015+
{
4016+
const auto castArray =
4017+
static_cast<const arrow::Decimal32Array *>(array);
4018+
if (!ConstraintEvaluator(
4019+
constraint,
4020+
CPLAtof(castArray->FormatValue(m_nIdxInBatch).c_str())))
4021+
{
4022+
return true;
4023+
}
4024+
break;
4025+
}
4026+
4027+
case arrow::Type::DECIMAL64:
4028+
{
4029+
const auto castArray =
4030+
static_cast<const arrow::Decimal64Array *>(array);
4031+
if (!ConstraintEvaluator(
4032+
constraint,
4033+
CPLAtof(castArray->FormatValue(m_nIdxInBatch).c_str())))
4034+
{
4035+
return true;
4036+
}
4037+
break;
4038+
}
4039+
#endif
4040+
39034041
case arrow::Type::DECIMAL128:
39044042
{
39054043
const auto castArray =

‎ogr/ogrsf_frmts/arrow_common/ograrrowwriterlayer.hpp

+13-1
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,19 @@ inline void OGRArrowWriterLayer::CreateSchemaCommon()
184184
{
185185
const int nPrecision = poFieldDefn->GetPrecision();
186186
if (nWidth != 0 && nPrecision != 0)
187-
dt = arrow::decimal(nWidth, nPrecision);
187+
{
188+
// Since arrow 18.0, we could use arrow::smallest_decimal()
189+
// to return the smallest representation (i.e. possibly
190+
// decimal32 and decimal64). But for now keep decimal128
191+
// as the minimum for backwards compatibility.
192+
// GetValueDecimal() and other functions in
193+
// ogrlayerarrow.cpp would have to be adapted for decimal32
194+
// and decimal64 compatibility.
195+
if (nWidth > 38)
196+
dt = arrow::decimal256(nWidth, nPrecision);
197+
else
198+
dt = arrow::decimal128(nWidth, nPrecision);
199+
}
188200
else if (eSubDT == OFSTFloat32)
189201
dt = arrow::float32();
190202
else

‎ogr/ogrsf_frmts/parquet/ogrparquetlayer.cpp

+4-1
Original file line numberDiff line numberDiff line change
@@ -1427,6 +1427,10 @@ bool OGRParquetLayer::ReadNextBatch()
14271427
{
14281428
m_nIdxInBatch = 0;
14291429

1430+
const int nNumGroups = m_poArrowReader->num_row_groups();
1431+
if (nNumGroups == 0)
1432+
return false;
1433+
14301434
if (m_bSingleBatch)
14311435
{
14321436
CPLAssert(m_iRecordBatch == 0);
@@ -1468,7 +1472,6 @@ bool OGRParquetLayer::ReadNextBatch()
14681472
}
14691473
else
14701474
{
1471-
const int nNumGroups = m_poArrowReader->num_row_groups();
14721475
OGRField sMin;
14731476
OGRField sMax;
14741477
OGR_RawField_SetNull(&sMin);

‎ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp

+14-1
Original file line numberDiff line numberDiff line change
@@ -1372,6 +1372,7 @@ bool OGRSQLiteBaseDataSource::OpenOrCreateDB(int flagsIn,
13721372
#endif
13731373

13741374
bool bPageSizeFound = false;
1375+
bool bSecureDeleteFound = false;
13751376

13761377
const char *pszSqlitePragma =
13771378
CPLGetConfigOption("OGR_SQLITE_PRAGMA", nullptr);
@@ -1456,7 +1457,7 @@ bool OGRSQLiteBaseDataSource::OpenOrCreateDB(int flagsIn,
14561457
{
14571458
if (STARTS_WITH_CI(papszTokens[i], "PAGE_SIZE"))
14581459
bPageSizeFound = true;
1459-
if (STARTS_WITH_CI(papszTokens[i], "JOURNAL_MODE"))
1460+
else if (STARTS_WITH_CI(papszTokens[i], "JOURNAL_MODE"))
14601461
{
14611462
const char *pszEqual = strchr(papszTokens[i], '=');
14621463
if (pszEqual)
@@ -1467,6 +1468,8 @@ bool OGRSQLiteBaseDataSource::OpenOrCreateDB(int flagsIn,
14671468
continue;
14681469
}
14691470
}
1471+
else if (STARTS_WITH_CI(papszTokens[i], "SECURE_DELETE"))
1472+
bSecureDeleteFound = true;
14701473

14711474
const char *pszSQL = CPLSPrintf("PRAGMA %s", papszTokens[i]);
14721475

@@ -1646,6 +1649,16 @@ bool OGRSQLiteBaseDataSource::OpenOrCreateDB(int flagsIn,
16461649
sqlite3_exec(hDB, pszSQL, nullptr, nullptr, nullptr));
16471650
}
16481651

1652+
if (!bSecureDeleteFound)
1653+
{
1654+
// Turn on secure_delete by default (unless the user specifies a
1655+
// value of this pragma through OGR_SQLITE_PRAGMA)
1656+
// For example, Debian and Conda-Forge SQLite3 builds already turn on
1657+
// secure_delete.
1658+
CPL_IGNORE_RET_VAL(sqlite3_exec(hDB, "PRAGMA secure_delete = 1",
1659+
nullptr, nullptr, nullptr));
1660+
}
1661+
16491662
SetCacheSize();
16501663
SetSynchronous();
16511664
if (bLoadExtensions)

0 commit comments

Comments
 (0)