Skip to content

Commit aa1c23b

Browse files
committed
Add datMap/datUnmap
Not supported by HDF5. Presumably because of things like compression filters and data type conversion. datMap must therefore mmap and anonymous memory area to receive the data and then datUnmap must copy the data back to the underlying dataset. Will require that datMap and datUnmap lose the const in the API as HDSLoc will need to be updated. Also need datAnnul to automatically do the copy. Unsolved problems (apart from no mmap): What happens if the primitive is mapped twice? With different locators? What happens in HDS? What happens if you call datPut/datGet whilst the locator is memory mapped? Do you have to track all the mapped locators and put some code in to try to unmap them all on program exit? If you annul the root locator should you go and annul all the child locators so that everything is synced before the file is closed?
1 parent a9f6343 commit aa1c23b

8 files changed

+661
-19
lines changed

Makefile.am

+3
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ C_ROUTINES = \
9393
datGet.c \
9494
datGet1C.c \
9595
datGetVC.c \
96+
datMap.c \
97+
datUnmap.c \
9698
datnew0.c \
9799
datnew1.c \
98100
datput1.c \
@@ -101,6 +103,7 @@ C_ROUTINES = \
101103
datget0x.c \
102104
datAnnul.c \
103105
dat1New.c \
106+
datmapx.c \
104107
dat1Type.c \
105108
dat1AllocLoc.c \
106109
dat1FreeLoc.c \

dat1.h

+23-9
Original file line numberDiff line numberDiff line change
@@ -73,18 +73,17 @@
7373
#include "hds1.h"
7474
#include "hds_types.h"
7575

76-
/* Private definition of the HDS locator struct */
77-
typedef struct LOC {
78-
hid_t file_id; /* Set if this locator is associated with a root file */
79-
hid_t dataset_id; /* Set if this is a dataset "primitive type" */
80-
hid_t dataspace_id; /* Set if this is a primitive with dimensions */
81-
hid_t group_id; /* Set if this locator is associated with a group */
82-
hid_t dtype; /* Set if a special data type was created for this locator */
83-
} HDSLoc;
76+
/* Access mode requested for a particular primitive type */
77+
typedef enum {
78+
HDSMODE_UNKNOWN = 0,
79+
HDSMODE_READ,
80+
HDSMODE_WRITE,
81+
HDSMODE_UPDATE
82+
} hdsmode_t;
8483

8584
/* All the standard HDS types as an enum. For internal use only. */
8685
typedef enum {
87-
HDSTYPE_NONE,
86+
HDSTYPE_NONE = 0,
8887
HDSTYPE_BYTE,
8988
HDSTYPE_UBYTE,
9089
HDSTYPE_WORD,
@@ -120,6 +119,21 @@ typedef enum {
120119
#define DAT__MXCHR 0xffff /* Max characters in a character data type */
121120

122121

122+
/* Private definition of the HDS locator struct */
123+
typedef struct LOC {
124+
void *pntr; /* Pointer to memory mapped data array [datMap only] */
125+
size_t bytesmapped; /* Number of bytes mapped into memory [datMap only] */
126+
hid_t file_id; /* Set if this locator is associated with a root file */
127+
hid_t dataset_id; /* Set if this is a dataset "primitive type" */
128+
hid_t dataspace_id; /* Set if this is a primitive with dimensions */
129+
hid_t group_id; /* Set if this locator is associated with a group */
130+
hid_t dtype; /* Set if a special data type was created for this locator */
131+
hdsmode_t accmode; /* Access mode for memory mapped data [datMap only] */
132+
int ndims; /* Number of dimensions in mapdims [datMap only] */
133+
hdsdim mapdims[DAT__MXDIM]; /* Dimensionality of mapped dims [datMap only] */
134+
char maptype[DAT__SZTYP+1]; /* HDS type string used for memory mapping [datMap only] */
135+
} HDSLoc;
136+
123137
/* Wrapper to check HDF5 status and map to EMS.
124138
Also does not call the routine unless status is good.
125139
Assumes inherited status is available in "status" and

datAnnul.c

+5
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
* History:
4242
* 2014-08-26 (TIMJ):
4343
* Initial version
44+
* 2014-08-29 (TIMJ):
45+
* datUnmap just in case.
4446
* {enter_further_changes_here}
4547
4648
* Copyright:
@@ -82,6 +84,9 @@ int datAnnul( HDSLoc **locator, int * status ) {
8284
if (!locator) return *status;
8385
if (! *locator) return *status;
8486

87+
/* Sort out any memory mapping */
88+
datUnmap( *locator, status );
89+
8590
thisloc = *locator;
8691

8792
if (thisloc->dtype) H5Tclose(thisloc->dtype);

datMap.c

+255
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
/*
2+
*+
3+
* Name:
4+
* datMap
5+
6+
* Purpose:
7+
* Map a primitive
8+
9+
* Language:
10+
* Starlink ANSI C
11+
12+
* Type of Module:
13+
* Library routine
14+
15+
* Invocation:
16+
* datMap (HDSLoc *locator, const char *type_str, const char *mode_str, int ndim,
17+
* const hdsdim dims[], void **pntr, int *status);
18+
19+
* Arguments:
20+
* locator = HDSLoc * (Given and Returned)
21+
* Primitive locator.
22+
* type_str = const char * (Given)
23+
* Data type
24+
* mode_str = const char * (Given)
25+
* Access mode (READ, UPDATE or WRITE)
26+
* ndim = int (Given)
27+
* Number of dimensions
28+
* dims = const hdsdim [] (Given)
29+
* Object dimensions.
30+
* pntr = void ** (Returned)
31+
* Pointer to be updated with the mapped data.
32+
* In WRITE mode the buffer will be filled with zeroes by the operating system.
33+
* In READ or UPDATE mode the buffer will contain the contents of the dataset.
34+
* status = int* (Given and Returned)
35+
* Pointer to global status.
36+
37+
* Description:
38+
* Map a primitive (access type specified by a parameter).
39+
40+
* Authors:
41+
* TIMJ: Tim Jenness (Cornell)
42+
* {enter_new_authors_here}
43+
44+
* Notes:
45+
* - The API differs slightly from the HDS API in that the locator
46+
* supplied to the routine is no longer marked const.
47+
* - HDF5 does not support memory mapping directly. This routine
48+
* therefore emulates memory mapping by allocating memory using
49+
* mmap() and, for read or update modes, reading the contents from
50+
* the HDF5 file.
51+
* - The resources will be freed either by using datUnmap or by
52+
* calling datAnnul().
53+
* - In WRITE and UPDATE mode the contents of the buffer will be
54+
* written to the HDF5 file on datUnmap() or datAnnul().
55+
* - The resultant pointer can be used from both C and Fortran
56+
* using CNF.
57+
58+
* History:
59+
* 2014-08-29 (TIMJ):
60+
* Initial version
61+
* {enter_further_changes_here}
62+
63+
* Copyright:
64+
* Copyright (C) 2014 Cornell University
65+
* All Rights Reserved.
66+
67+
* Licence:
68+
* Redistribution and use in source and binary forms, with or
69+
* without modification, are permitted provided that the following
70+
* conditions are met:
71+
*
72+
* - Redistributions of source code must retain the above copyright
73+
* notice, this list of conditions and the following disclaimer.
74+
*
75+
* - Redistributions in binary form must reproduce the above
76+
* copyright notice, this list of conditions and the following
77+
* disclaimer in the documentation and/or other materials
78+
* provided with the distribution.
79+
*
80+
* - Neither the name of the {organization} nor the names of its
81+
* contributors may be used to endorse or promote products
82+
* derived from this software without specific prior written
83+
* permission.
84+
*
85+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
86+
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
87+
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
88+
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
89+
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
90+
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
91+
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
92+
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
93+
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
94+
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
95+
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
96+
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
97+
* THE POSSIBILITY OF SUCH DAMAGE.
98+
99+
* Bugs:
100+
* {note_any_bugs_here}
101+
*-
102+
*/
103+
104+
#include <errno.h>
105+
#include <sys/mman.h>
106+
107+
#include "hdf5.h"
108+
#include "hdf5_hl.h"
109+
110+
#include "star/util.h"
111+
#include "ems.h"
112+
#include "sae_par.h"
113+
114+
#include "hds1.h"
115+
#include "dat1.h"
116+
#include "hds.h"
117+
118+
#include "dat_err.h"
119+
120+
#include "f77.h"
121+
122+
int
123+
datMap(HDSLoc *locator, const char *type_str, const char *mode_str, int ndim,
124+
const hdsdim dims[], void **pntr, int *status) {
125+
126+
int isprim = 0;
127+
char normtypestr[DAT__SZTYP+1];
128+
size_t nbytes = 0;
129+
hid_t h5type = 0;
130+
int typcreat = 0;
131+
void *mapped = NULL;
132+
int isreg = 0;
133+
hdsmode_t accmode = HDSMODE_UNKNOWN;
134+
135+
if (*status != SAI__OK) return *status;
136+
137+
/* Get the HDF5 type code and confirm this is a primitive type */
138+
isprim = dau1CheckType( type_str, &h5type, normtypestr,
139+
sizeof(normtypestr), &typcreat, status );
140+
141+
if (!isprim) {
142+
if (*status == SAI__OK) {
143+
*status = DAT__TYPIN;
144+
emsRepf("datMap_1", "datGet: Data type must be a primitive type and not '%s'",
145+
status, normtypestr);
146+
}
147+
goto CLEANUP;
148+
}
149+
150+
/* Now we want the HDSTYPE of the requested type so that we can work out how much
151+
memory we will need to allocate. For that we can use dat1Type */
152+
nbytes = H5Tget_size( h5type );
153+
154+
{
155+
int i;
156+
if (ndim > 0) {
157+
for (i = 0; i < ndim; i++) {
158+
nbytes *= dims[i];
159+
}
160+
}
161+
}
162+
163+
/* Get some anonymous memory - we always have to map read/write
164+
because we always have to copy data into this space. */
165+
mapped = mmap( NULL, nbytes, PROT_READ|PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0 );
166+
if (mapped == MAP_FAILED) {
167+
emsSyser( "MESSAGE", errno );
168+
*status = DAT__FILMP;
169+
emsRep("datMap_2", "Error mapping some memory: ^MESSAGE", status );
170+
mapped = NULL;
171+
goto CLEANUP;
172+
}
173+
174+
/* Must register this with CNF */
175+
if (*status != SAI__OK) goto CLEANUP;
176+
177+
isreg = cnfRegp( mapped );
178+
if (isreg == -1) {
179+
/* Series internal error */
180+
*status = DAT__FILMP;
181+
emsRep("datMap_3", "Error registering a pointer for mapped data "
182+
" - internal CNF error", status );
183+
goto CLEANUP;
184+
} else if (isreg == 0) {
185+
/* HDS will tweak the offset until registration is possible */
186+
*status = DAT__FILMP;
187+
emsRep("datMap_5", "Unable to register mapped pointer with CNF",
188+
status);
189+
goto CLEANUP;
190+
}
191+
192+
if (*status != SAI__OK) goto CLEANUP;
193+
194+
/* Now we have some memory we need to decide if we are filling it
195+
or note */
196+
197+
switch (mode_str[0]) {
198+
case 'R':
199+
accmode = HDSMODE_READ;
200+
case 'U':
201+
datGet( locator, type_str, ndim, dims, mapped, status );
202+
if (accmode == HDSMODE_UNKNOWN) accmode = HDSMODE_UPDATE; /* Prevent bad R case */
203+
break;
204+
case 'W':
205+
accmode = HDSMODE_WRITE;
206+
break;
207+
default:
208+
*status = DAT__FILMP;
209+
emsRepf("datMap_6", "Unrecognized mode string '%s' for datMap",
210+
status, mode_str);
211+
}
212+
213+
/* Need to store the mapped information in the locator so that
214+
we can unmap it later */
215+
if (*status != SAI__OK) goto CLEANUP;
216+
217+
CLEANUP:
218+
/* Cleanups that must happen always */
219+
if (h5type && typcreat) H5Tclose(h5type);
220+
221+
/* cleanups that only happen if status is bad */
222+
if (*status != SAI__OK) {
223+
if (mapped) {
224+
if (isreg == 1) cnfUregp( mapped );
225+
if ( munmap( mapped, nbytes ) != 0 ) {
226+
emsSyser( "MESSAGE", errno );
227+
emsRep("datMap_4", "Error unmapping mapped memory: ^MESSAGE", status);
228+
}
229+
mapped = NULL;
230+
}
231+
}
232+
233+
/* Update the locator to reflect the mapped status */
234+
if (*status == SAI__OK) {
235+
int i;
236+
locator->pntr = mapped;
237+
locator->bytesmapped = nbytes;
238+
locator->accmode = accmode;
239+
240+
/* In order to copy the data back into the underlying HDF5 dataset
241+
we need to store additional information about how this was mapped
242+
to allow us to either call datPut later on or at least a new
243+
dataspace. For now store the arguments so we can pass them straight
244+
to datPut */
245+
locator->ndims = ndim;
246+
for (i=0; i<ndim; i++) {
247+
(locator->mapdims)[i] = dims[i];
248+
}
249+
star_strlcpy( locator->maptype, normtypestr, sizeof(locator->maptype) );
250+
}
251+
252+
*pntr = locator->pntr;
253+
254+
return *status;
255+
}

0 commit comments

Comments
 (0)