nostrdb

an unfairly fast embedded nostr database backed by lmdb
git clone git://jb55.com/nostrdb
Log | Files | Refs | Submodules | README | LICENSE

mdb.c (292667B)


      1 /** @file mdb.c
      2  *	@brief Lightning memory-mapped database library
      3  *
      4  *	A Btree-based database management library modeled loosely on the
      5  *	BerkeleyDB API, but much simplified.
      6  */
      7 /*
      8  * Copyright 2011-2021 Howard Chu, Symas Corp.
      9  * All rights reserved.
     10  *
     11  * Redistribution and use in source and binary forms, with or without
     12  * modification, are permitted only as authorized by the OpenLDAP
     13  * Public License.
     14  *
     15  * A copy of this license is available in the file LICENSE in the
     16  * top-level directory of the distribution or, alternatively, at
     17  * <http://www.OpenLDAP.org/license.html>.
     18  *
     19  * This code is derived from btree.c written by Martin Hedenfalk.
     20  *
     21  * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se>
     22  *
     23  * Permission to use, copy, modify, and distribute this software for any
     24  * purpose with or without fee is hereby granted, provided that the above
     25  * copyright notice and this permission notice appear in all copies.
     26  *
     27  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     28  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     29  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     30  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     31  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     32  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     33  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     34  */
     35 #ifndef _GNU_SOURCE
     36 #define _GNU_SOURCE 1
     37 #endif
     38 #if defined(__WIN64__)
     39 #define _FILE_OFFSET_BITS	64
     40 #endif
     41 #ifdef _WIN32
     42 #include <malloc.h>
     43 #include <windows.h>
     44 #include <wchar.h>				/* get wcscpy() */
     45 
     46 /** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it
     47  *  as int64 which is wrong. MSVC doesn't define it at all, so just
     48  *  don't use it.
     49  */
     50 #define MDB_PID_T	int
     51 #define MDB_THR_T	DWORD
     52 #include <sys/types.h>
     53 #include <sys/stat.h>
     54 #ifdef __GNUC__
     55 # include <sys/param.h>
     56 #else
     57 # define LITTLE_ENDIAN	1234
     58 # define BIG_ENDIAN	4321
     59 # define BYTE_ORDER	LITTLE_ENDIAN
     60 # ifndef SSIZE_MAX
     61 #  define SSIZE_MAX	INT_MAX
     62 # endif
     63 #endif
     64 #else
     65 #include <sys/types.h>
     66 #include <sys/stat.h>
     67 #define MDB_PID_T	pid_t
     68 #define MDB_THR_T	pthread_t
     69 #include <sys/param.h>
     70 #include <sys/uio.h>
     71 #include <sys/mman.h>
     72 #ifdef HAVE_SYS_FILE_H
     73 #include <sys/file.h>
     74 #endif
     75 #include <fcntl.h>
     76 #endif
     77 
     78 #if defined(__mips) && defined(__linux)
     79 /* MIPS has cache coherency issues, requires explicit cache control */
     80 #include <sys/cachectl.h>
     81 #define CACHEFLUSH(addr, bytes, cache)	cacheflush(addr, bytes, cache)
     82 #else
     83 #define CACHEFLUSH(addr, bytes, cache)
     84 #endif
     85 
     86 #if defined(__linux) && !defined(MDB_FDATASYNC_WORKS)
     87 /** fdatasync is broken on ext3/ext4fs on older kernels, see
     88  *	description in #mdb_env_open2 comments. You can safely
     89  *	define MDB_FDATASYNC_WORKS if this code will only be run
     90  *	on kernels 3.6 and newer.
     91  */
     92 #define	BROKEN_FDATASYNC
     93 #endif
     94 
     95 #include <errno.h>
     96 #include <limits.h>
     97 #include <stddef.h>
     98 #include <inttypes.h>
     99 #include <stdio.h>
    100 #include <stdlib.h>
    101 #include <string.h>
    102 #include <time.h>
    103 
    104 #ifdef _MSC_VER
    105 #include <io.h>
    106 typedef SSIZE_T	ssize_t;
    107 #else
    108 #include <unistd.h>
    109 #endif
    110 
    111 #if defined(__sun) || defined(ANDROID)
    112 /* Most platforms have posix_memalign, older may only have memalign */
    113 #define HAVE_MEMALIGN	1
    114 #include <malloc.h>
    115 /* On Solaris, we need the POSIX sigwait function */
    116 #if defined (__sun)
    117 # define _POSIX_PTHREAD_SEMANTICS	1
    118 #endif
    119 #endif
    120 
    121 #if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER))
    122 #include <netinet/in.h>
    123 #include <resolv.h>	/* defines BYTE_ORDER on HPUX and Solaris */
    124 #endif
    125 
    126 #if defined(__FreeBSD__) && defined(__FreeBSD_version) && __FreeBSD_version >= 1100110
    127 # define MDB_USE_POSIX_MUTEX	1
    128 # define MDB_USE_ROBUST	1
    129 #elif defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__)
    130 # define MDB_USE_POSIX_SEM	1
    131 # define MDB_FDATASYNC		fsync
    132 #elif defined(ANDROID)
    133 # define MDB_FDATASYNC		fsync
    134 #endif
    135 
    136 #ifndef _WIN32
    137 #include <pthread.h>
    138 #include <signal.h>
    139 #ifdef MDB_USE_POSIX_SEM
    140 # define MDB_USE_HASH		1
    141 #include <semaphore.h>
    142 #else
    143 #define MDB_USE_POSIX_MUTEX	1
    144 #endif
    145 #endif
    146 
    147 #if defined(_WIN32) + defined(MDB_USE_POSIX_SEM) \
    148 	+ defined(MDB_USE_POSIX_MUTEX) != 1
    149 # error "Ambiguous shared-lock implementation"
    150 #endif
    151 
    152 #ifdef USE_VALGRIND
    153 #include <valgrind/memcheck.h>
    154 #define VGMEMP_CREATE(h,r,z)    VALGRIND_CREATE_MEMPOOL(h,r,z)
    155 #define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s)
    156 #define VGMEMP_FREE(h,a) VALGRIND_MEMPOOL_FREE(h,a)
    157 #define VGMEMP_DESTROY(h)	VALGRIND_DESTROY_MEMPOOL(h)
    158 #define VGMEMP_DEFINED(a,s)	VALGRIND_MAKE_MEM_DEFINED(a,s)
    159 #else
    160 #define VGMEMP_CREATE(h,r,z)
    161 #define VGMEMP_ALLOC(h,a,s)
    162 #define VGMEMP_FREE(h,a)
    163 #define VGMEMP_DESTROY(h)
    164 #define VGMEMP_DEFINED(a,s)
    165 #endif
    166 
    167 #ifndef BYTE_ORDER
    168 # if (defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN))
    169 /* Solaris just defines one or the other */
    170 #  define LITTLE_ENDIAN	1234
    171 #  define BIG_ENDIAN	4321
    172 #  ifdef _LITTLE_ENDIAN
    173 #   define BYTE_ORDER  LITTLE_ENDIAN
    174 #  else
    175 #   define BYTE_ORDER  BIG_ENDIAN
    176 #  endif
    177 # else
    178 #  define BYTE_ORDER   __BYTE_ORDER
    179 # endif
    180 #endif
    181 
    182 #ifndef LITTLE_ENDIAN
    183 #define LITTLE_ENDIAN	__LITTLE_ENDIAN
    184 #endif
    185 #ifndef BIG_ENDIAN
    186 #define BIG_ENDIAN	__BIG_ENDIAN
    187 #endif
    188 
    189 #if defined(__i386) || defined(__x86_64) || defined(_M_IX86)
    190 #define MISALIGNED_OK	1
    191 #endif
    192 
    193 #include "lmdb.h"
    194 #include "midl.h"
    195 
    196 #if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN)
    197 # error "Unknown or unsupported endianness (BYTE_ORDER)"
    198 #elif (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
    199 # error "Two's complement, reasonably sized integer types, please"
    200 #endif
    201 
    202 #if (((__clang_major__ << 8) | __clang_minor__) >= 0x0302) || (((__GNUC__ << 8) | __GNUC_MINOR__) >= 0x0403)
    203 /** Mark infrequently used env functions as cold. This puts them in a separate
    204  *  section, and optimizes them for size */
    205 #define ESECT __attribute__ ((cold))
    206 #else
    207 /* On older compilers, use a separate section */
    208 # ifdef __GNUC__
    209 #  ifdef __APPLE__
    210 #   define      ESECT   __attribute__ ((section("__TEXT,text_env")))
    211 #  else
    212 #   define      ESECT   __attribute__ ((section("text_env")))
    213 #  endif
    214 # else
    215 #  define ESECT
    216 # endif
    217 #endif
    218 
    219 #ifdef _WIN32
    220 #define CALL_CONV WINAPI
    221 #else
    222 #define CALL_CONV
    223 #endif
    224 
    225 /** @defgroup internal	LMDB Internals
    226  *	@{
    227  */
    228 /** @defgroup compat	Compatibility Macros
    229  *	A bunch of macros to minimize the amount of platform-specific ifdefs
    230  *	needed throughout the rest of the code. When the features this library
    231  *	needs are similar enough to POSIX to be hidden in a one-or-two line
    232  *	replacement, this macro approach is used.
    233  *	@{
    234  */
    235 
    236 	/** Features under development */
    237 #ifndef MDB_DEVEL
    238 #define MDB_DEVEL 0
    239 #endif
    240 
    241 	/** Wrapper around __func__, which is a C99 feature */
    242 #if __STDC_VERSION__ >= 199901L
    243 # define mdb_func_	__func__
    244 #elif __GNUC__ >= 2 || _MSC_VER >= 1300
    245 # define mdb_func_	__FUNCTION__
    246 #else
    247 /* If a debug message says <mdb_unknown>(), update the #if statements above */
    248 # define mdb_func_	"<mdb_unknown>"
    249 #endif
    250 
    251 /* Internal error codes, not exposed outside liblmdb */
    252 #define	MDB_NO_ROOT		(MDB_LAST_ERRCODE + 10)
    253 #ifdef _WIN32
    254 #define MDB_OWNERDEAD	((int) WAIT_ABANDONED)
    255 #elif defined(MDB_USE_POSIX_MUTEX) && defined(EOWNERDEAD)
    256 #define MDB_OWNERDEAD	EOWNERDEAD	/**< #LOCK_MUTEX0() result if dead owner */
    257 #endif
    258 
    259 #ifdef __GLIBC__
    260 #define	GLIBC_VER	((__GLIBC__ << 16 )| __GLIBC_MINOR__)
    261 #endif
    262 /** Some platforms define the EOWNERDEAD error code
    263  * even though they don't support Robust Mutexes.
    264  * Compile with -DMDB_USE_ROBUST=0, or use some other
    265  * mechanism like -DMDB_USE_POSIX_SEM instead of
    266  * -DMDB_USE_POSIX_MUTEX.
    267  * (Posix semaphores are not robust.)
    268  */
    269 #ifndef MDB_USE_ROBUST
    270 /* Android currently lacks Robust Mutex support. So does glibc < 2.4. */
    271 # if defined(MDB_USE_POSIX_MUTEX) && (defined(ANDROID) || \
    272 	(defined(__GLIBC__) && GLIBC_VER < 0x020004))
    273 #  define MDB_USE_ROBUST	0
    274 # else
    275 #  define MDB_USE_ROBUST	1
    276 # endif
    277 #endif /* !MDB_USE_ROBUST */
    278 
    279 #if defined(MDB_USE_POSIX_MUTEX) && (MDB_USE_ROBUST)
    280 /* glibc < 2.12 only provided _np API */
    281 #  if (defined(__GLIBC__) && GLIBC_VER < 0x02000c) || \
    282 	(defined(PTHREAD_MUTEX_ROBUST_NP) && !defined(PTHREAD_MUTEX_ROBUST))
    283 #   define PTHREAD_MUTEX_ROBUST	PTHREAD_MUTEX_ROBUST_NP
    284 #   define pthread_mutexattr_setrobust(attr, flag)	pthread_mutexattr_setrobust_np(attr, flag)
    285 #   define pthread_mutex_consistent(mutex)	pthread_mutex_consistent_np(mutex)
    286 #  endif
    287 #endif /* MDB_USE_POSIX_MUTEX && MDB_USE_ROBUST */
    288 
    289 #if defined(MDB_OWNERDEAD) && (MDB_USE_ROBUST)
    290 #define MDB_ROBUST_SUPPORTED	1
    291 #endif
    292 
    293 #ifdef _WIN32
    294 #define MDB_USE_HASH	1
    295 #define MDB_PIDLOCK	0
    296 #define THREAD_RET	DWORD
    297 #define pthread_t	HANDLE
    298 #define pthread_mutex_t	HANDLE
    299 #define pthread_cond_t	HANDLE
    300 typedef HANDLE mdb_mutex_t, mdb_mutexref_t;
    301 #define pthread_key_t	DWORD
    302 #define pthread_self()	GetCurrentThreadId()
    303 #define pthread_key_create(x,y)	\
    304 	((*(x) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? ErrCode() : 0)
    305 #define pthread_key_delete(x)	TlsFree(x)
    306 #define pthread_getspecific(x)	TlsGetValue(x)
    307 #define pthread_setspecific(x,y)	(TlsSetValue(x,y) ? 0 : ErrCode())
    308 #define pthread_mutex_unlock(x)	ReleaseMutex(*x)
    309 #define pthread_mutex_lock(x)	WaitForSingleObject(*x, INFINITE)
    310 #define pthread_cond_signal(x)	SetEvent(*x)
    311 #define pthread_cond_wait(cond,mutex)	do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0)
    312 #define THREAD_CREATE(thr,start,arg) \
    313 	(((thr) = CreateThread(NULL, 0, start, arg, 0, NULL)) ? 0 : ErrCode())
    314 #define THREAD_FINISH(thr) \
    315 	(WaitForSingleObject(thr, INFINITE) ? ErrCode() : 0)
    316 #define LOCK_MUTEX0(mutex)		WaitForSingleObject(mutex, INFINITE)
    317 #define UNLOCK_MUTEX(mutex)		ReleaseMutex(mutex)
    318 #define mdb_mutex_consistent(mutex)	0
    319 #define getpid()	GetCurrentProcessId()
    320 #define	MDB_FDATASYNC(fd)	(!FlushFileBuffers(fd))
    321 #define	MDB_MSYNC(addr,len,flags)	(!FlushViewOfFile(addr,len))
    322 #define	ErrCode()	GetLastError()
    323 #define GET_PAGESIZE(x) {SYSTEM_INFO si; GetSystemInfo(&si); (x) = si.dwPageSize;}
    324 #define	close(fd)	(CloseHandle(fd) ? 0 : -1)
    325 #define	munmap(ptr,len)	UnmapViewOfFile(ptr)
    326 #ifdef PROCESS_QUERY_LIMITED_INFORMATION
    327 #define MDB_PROCESS_QUERY_LIMITED_INFORMATION PROCESS_QUERY_LIMITED_INFORMATION
    328 #else
    329 #define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000
    330 #endif
    331 #define	Z	"I"
    332 #else
    333 #define THREAD_RET	void *
    334 #define THREAD_CREATE(thr,start,arg)	pthread_create(&thr,NULL,start,arg)
    335 #define THREAD_FINISH(thr)	pthread_join(thr,NULL)
    336 #define	Z	"z"			/**< printf format modifier for size_t */
    337 
    338 	/** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */
    339 #define MDB_PIDLOCK			1
    340 
    341 #ifdef MDB_USE_POSIX_SEM
    342 
    343 typedef sem_t *mdb_mutex_t, *mdb_mutexref_t;
    344 #define LOCK_MUTEX0(mutex)		mdb_sem_wait(mutex)
    345 #define UNLOCK_MUTEX(mutex)		sem_post(mutex)
    346 
    347 static int
    348 mdb_sem_wait(sem_t *sem)
    349 {
    350    int rc;
    351    while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ;
    352    return rc;
    353 }
    354 
    355 #else	/* MDB_USE_POSIX_MUTEX: */
    356 	/** Shared mutex/semaphore as the original is stored.
    357 	 *
    358 	 *	Not for copies.  Instead it can be assigned to an #mdb_mutexref_t.
    359 	 *	When mdb_mutexref_t is a pointer and mdb_mutex_t is not, then it
    360 	 *	is array[size 1] so it can be assigned to the pointer.
    361 	 */
    362 typedef pthread_mutex_t mdb_mutex_t[1];
    363 	/** Reference to an #mdb_mutex_t */
    364 typedef pthread_mutex_t *mdb_mutexref_t;
    365 	/** Lock the reader or writer mutex.
    366 	 *	Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX().
    367 	 */
    368 #define LOCK_MUTEX0(mutex)	pthread_mutex_lock(mutex)
    369 	/** Unlock the reader or writer mutex.
    370 	 */
    371 #define UNLOCK_MUTEX(mutex)	pthread_mutex_unlock(mutex)
    372 	/** Mark mutex-protected data as repaired, after death of previous owner.
    373 	 */
    374 #define mdb_mutex_consistent(mutex)	pthread_mutex_consistent(mutex)
    375 #endif	/* MDB_USE_POSIX_SEM */
    376 
    377 	/** Get the error code for the last failed system function.
    378 	 */
    379 #define	ErrCode()	errno
    380 
    381 	/** An abstraction for a file handle.
    382 	 *	On POSIX systems file handles are small integers. On Windows
    383 	 *	they're opaque pointers.
    384 	 */
    385 #define	HANDLE	int
    386 
    387 	/**	A value for an invalid file handle.
    388 	 *	Mainly used to initialize file variables and signify that they are
    389 	 *	unused.
    390 	 */
    391 #define INVALID_HANDLE_VALUE	(-1)
    392 
    393 	/** Get the size of a memory page for the system.
    394 	 *	This is the basic size that the platform's memory manager uses, and is
    395 	 *	fundamental to the use of memory-mapped files.
    396 	 */
    397 #define	GET_PAGESIZE(x)	((x) = sysconf(_SC_PAGE_SIZE))
    398 #endif
    399 
    400 #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
    401 #define MNAME_LEN	32
    402 #else
    403 #define MNAME_LEN	(sizeof(pthread_mutex_t))
    404 #endif
    405 
    406 /** @} */
    407 
    408 #ifdef MDB_ROBUST_SUPPORTED
    409 	/** Lock mutex, handle any error, set rc = result.
    410 	 *	Return 0 on success, nonzero (not rc) on error.
    411 	 */
    412 #define LOCK_MUTEX(rc, env, mutex) \
    413 	(((rc) = LOCK_MUTEX0(mutex)) && \
    414 	 ((rc) = mdb_mutex_failed(env, mutex, rc)))
    415 static int mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc);
    416 #else
    417 #define LOCK_MUTEX(rc, env, mutex) ((rc) = LOCK_MUTEX0(mutex))
    418 #define mdb_mutex_failed(env, mutex, rc) (rc)
    419 #endif
    420 
    421 #ifndef _WIN32
    422 /**	A flag for opening a file and requesting synchronous data writes.
    423  *	This is only used when writing a meta page. It's not strictly needed;
    424  *	we could just do a normal write and then immediately perform a flush.
    425  *	But if this flag is available it saves us an extra system call.
    426  *
    427  *	@note If O_DSYNC is undefined but exists in /usr/include,
    428  * preferably set some compiler flag to get the definition.
    429  */
    430 #ifndef MDB_DSYNC
    431 # ifdef O_DSYNC
    432 # define MDB_DSYNC	O_DSYNC
    433 # else
    434 # define MDB_DSYNC	O_SYNC
    435 # endif
    436 #endif
    437 #endif
    438 
    439 /** Function for flushing the data of a file. Define this to fsync
    440  *	if fdatasync() is not supported.
    441  */
    442 #ifndef MDB_FDATASYNC
    443 # define MDB_FDATASYNC	fdatasync
    444 #endif
    445 
    446 #ifndef MDB_MSYNC
    447 # define MDB_MSYNC(addr,len,flags)	msync(addr,len,flags)
    448 #endif
    449 
    450 #ifndef MS_SYNC
    451 #define	MS_SYNC	1
    452 #endif
    453 
    454 #ifndef MS_ASYNC
    455 #define	MS_ASYNC	0
    456 #endif
    457 
    458 	/** A page number in the database.
    459 	 *	Note that 64 bit page numbers are overkill, since pages themselves
    460 	 *	already represent 12-13 bits of addressable memory, and the OS will
    461 	 *	always limit applications to a maximum of 63 bits of address space.
    462 	 *
    463 	 *	@note In the #MDB_node structure, we only store 48 bits of this value,
    464 	 *	which thus limits us to only 60 bits of addressable data.
    465 	 */
    466 typedef MDB_ID	pgno_t;
    467 
    468 	/** A transaction ID.
    469 	 *	See struct MDB_txn.mt_txnid for details.
    470 	 */
    471 typedef MDB_ID	txnid_t;
    472 
    473 /** @defgroup debug	Debug Macros
    474  *	@{
    475  */
    476 #ifndef MDB_DEBUG
    477 	/**	Enable debug output.  Needs variable argument macros (a C99 feature).
    478 	 *	Set this to 1 for copious tracing. Set to 2 to add dumps of all IDLs
    479 	 *	read from and written to the database (used for free space management).
    480 	 */
    481 #define MDB_DEBUG 0
    482 #endif
    483 
    484 #if MDB_DEBUG
    485 static int mdb_debug;
    486 static txnid_t mdb_debug_start;
    487 
    488 	/**	Print a debug message with printf formatting.
    489 	 *	Requires double parenthesis around 2 or more args.
    490 	 */
    491 # define DPRINTF(args) ((void) ((mdb_debug) && DPRINTF0 args))
    492 # define DPRINTF0(fmt, ...) \
    493 	fprintf(stderr, "%s:%d " fmt "\n", mdb_func_, __LINE__, __VA_ARGS__)
    494 #else
    495 # define DPRINTF(args)	((void) 0)
    496 #endif
    497 	/**	Print a debug string.
    498 	 *	The string is printed literally, with no format processing.
    499 	 */
    500 #define DPUTS(arg)	DPRINTF(("%s", arg))
    501 	/** Debugging output value of a cursor DBI: Negative in a sub-cursor. */
    502 #define DDBI(mc) \
    503 	(((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
    504 /** @} */
    505 
    506 	/**	@brief The maximum size of a database page.
    507 	 *
    508 	 *	It is 32k or 64k, since value-PAGEBASE must fit in
    509 	 *	#MDB_page.%mp_upper.
    510 	 *
    511 	 *	LMDB will use database pages < OS pages if needed.
    512 	 *	That causes more I/O in write transactions: The OS must
    513 	 *	know (read) the whole page before writing a partial page.
    514 	 *
    515 	 *	Note that we don't currently support Huge pages. On Linux,
    516 	 *	regular data files cannot use Huge pages, and in general
    517 	 *	Huge pages aren't actually pageable. We rely on the OS
    518 	 *	demand-pager to read our data and page it out when memory
    519 	 *	pressure from other processes is high. So until OSs have
    520 	 *	actual paging support for Huge pages, they're not viable.
    521 	 */
    522 #define MAX_PAGESIZE	 (PAGEBASE ? 0x10000 : 0x8000)
    523 
    524 	/** The minimum number of keys required in a database page.
    525 	 *	Setting this to a larger value will place a smaller bound on the
    526 	 *	maximum size of a data item. Data items larger than this size will
    527 	 *	be pushed into overflow pages instead of being stored directly in
    528 	 *	the B-tree node. This value used to default to 4. With a page size
    529 	 *	of 4096 bytes that meant that any item larger than 1024 bytes would
    530 	 *	go into an overflow page. That also meant that on average 2-3KB of
    531 	 *	each overflow page was wasted space. The value cannot be lower than
    532 	 *	2 because then there would no longer be a tree structure. With this
    533 	 *	value, items larger than 2KB will go into overflow pages, and on
    534 	 *	average only 1KB will be wasted.
    535 	 */
    536 #define MDB_MINKEYS	 2
    537 
    538 	/**	A stamp that identifies a file as an LMDB file.
    539 	 *	There's nothing special about this value other than that it is easily
    540 	 *	recognizable, and it will reflect any byte order mismatches.
    541 	 */
    542 #define MDB_MAGIC	 0xBEEFC0DE
    543 
    544 	/**	The version number for a database's datafile format. */
    545 #define MDB_DATA_VERSION	 ((MDB_DEVEL) ? 999 : 1)
    546 	/**	The version number for a database's lockfile format. */
    547 #define MDB_LOCK_VERSION	 1
    548 
    549 	/**	@brief The max size of a key we can write, or 0 for computed max.
    550 	 *
    551 	 *	This macro should normally be left alone or set to 0.
    552 	 *	Note that a database with big keys or dupsort data cannot be
    553 	 *	reliably modified by a liblmdb which uses a smaller max.
    554 	 *	The default is 511 for backwards compat, or 0 when #MDB_DEVEL.
    555 	 *
    556 	 *	Other values are allowed, for backwards compat.  However:
    557 	 *	A value bigger than the computed max can break if you do not
    558 	 *	know what you are doing, and liblmdb <= 0.9.10 can break when
    559 	 *	modifying a DB with keys/dupsort data bigger than its max.
    560 	 *
    561 	 *	Data items in an #MDB_DUPSORT database are also limited to
    562 	 *	this size, since they're actually keys of a sub-DB.  Keys and
    563 	 *	#MDB_DUPSORT data items must fit on a node in a regular page.
    564 	 */
    565 #ifndef MDB_MAXKEYSIZE
    566 #define MDB_MAXKEYSIZE	 ((MDB_DEVEL) ? 0 : 511)
    567 #endif
    568 
    569 	/**	The maximum size of a key we can write to the environment. */
    570 #if MDB_MAXKEYSIZE
    571 #define ENV_MAXKEY(env)	(MDB_MAXKEYSIZE)
    572 #else
    573 #define ENV_MAXKEY(env)	((env)->me_maxkey)
    574 #endif
    575 
    576 	/**	@brief The maximum size of a data item.
    577 	 *
    578 	 *	We only store a 32 bit value for node sizes.
    579 	 */
    580 #define MAXDATASIZE	0xffffffffUL
    581 
    582 #if MDB_DEBUG
    583 	/**	Key size which fits in a #DKBUF.
    584 	 *	@ingroup debug
    585 	 */
    586 #define DKBUF_MAXKEYSIZE ((MDB_MAXKEYSIZE) > 0 ? (MDB_MAXKEYSIZE) : 511)
    587 	/**	A key buffer.
    588 	 *	@ingroup debug
    589 	 *	This is used for printing a hex dump of a key's contents.
    590 	 */
    591 #define DKBUF	char kbuf[DKBUF_MAXKEYSIZE*2+1]
    592 	/**	Display a key in hex.
    593 	 *	@ingroup debug
    594 	 *	Invoke a function to display a key in hex.
    595 	 */
    596 #define	DKEY(x)	mdb_dkey(x, kbuf)
    597 #else
    598 #define	DKBUF
    599 #define DKEY(x)	0
    600 #endif
    601 
    602 	/** An invalid page number.
    603 	 *	Mainly used to denote an empty tree.
    604 	 */
    605 #define P_INVALID	 (~(pgno_t)0)
    606 
    607 	/** Test if the flags \b f are set in a flag word \b w. */
    608 #define F_ISSET(w, f)	 (((w) & (f)) == (f))
    609 
    610 	/** Round \b n up to an even number. */
    611 #define EVEN(n)		(((n) + 1U) & -2) /* sign-extending -2 to match n+1U */
    612 
    613 	/**	Used for offsets within a single page.
    614 	 *	Since memory pages are typically 4 or 8KB in size, 12-13 bits,
    615 	 *	this is plenty.
    616 	 */
    617 typedef uint16_t	 indx_t;
    618 
    619 	/**	Default size of memory map.
    620 	 *	This is certainly too small for any actual applications. Apps should always set
    621 	 *	the size explicitly using #mdb_env_set_mapsize().
    622 	 */
    623 #define DEFAULT_MAPSIZE	1048576
    624 
    625 /**	@defgroup readers	Reader Lock Table
    626  *	Readers don't acquire any locks for their data access. Instead, they
    627  *	simply record their transaction ID in the reader table. The reader
    628  *	mutex is needed just to find an empty slot in the reader table. The
    629  *	slot's address is saved in thread-specific data so that subsequent read
    630  *	transactions started by the same thread need no further locking to proceed.
    631  *
    632  *	If #MDB_NOTLS is set, the slot address is not saved in thread-specific data.
    633  *
    634  *	No reader table is used if the database is on a read-only filesystem, or
    635  *	if #MDB_NOLOCK is set.
    636  *
    637  *	Since the database uses multi-version concurrency control, readers don't
    638  *	actually need any locking. This table is used to keep track of which
    639  *	readers are using data from which old transactions, so that we'll know
    640  *	when a particular old transaction is no longer in use. Old transactions
    641  *	that have discarded any data pages can then have those pages reclaimed
    642  *	for use by a later write transaction.
    643  *
    644  *	The lock table is constructed such that reader slots are aligned with the
    645  *	processor's cache line size. Any slot is only ever used by one thread.
    646  *	This alignment guarantees that there will be no contention or cache
    647  *	thrashing as threads update their own slot info, and also eliminates
    648  *	any need for locking when accessing a slot.
    649  *
    650  *	A writer thread will scan every slot in the table to determine the oldest
    651  *	outstanding reader transaction. Any freed pages older than this will be
    652  *	reclaimed by the writer. The writer doesn't use any locks when scanning
    653  *	this table. This means that there's no guarantee that the writer will
    654  *	see the most up-to-date reader info, but that's not required for correct
    655  *	operation - all we need is to know the upper bound on the oldest reader,
    656  *	we don't care at all about the newest reader. So the only consequence of
    657  *	reading stale information here is that old pages might hang around a
    658  *	while longer before being reclaimed. That's actually good anyway, because
    659  *	the longer we delay reclaiming old pages, the more likely it is that a
    660  *	string of contiguous pages can be found after coalescing old pages from
    661  *	many old transactions together.
    662  *	@{
    663  */
    664 	/**	Number of slots in the reader table.
    665 	 *	This value was chosen somewhat arbitrarily. 126 readers plus a
    666 	 *	couple mutexes fit exactly into 8KB on my development machine.
    667 	 *	Applications should set the table size using #mdb_env_set_maxreaders().
    668 	 */
    669 #define DEFAULT_READERS	126
    670 
    671 	/**	The size of a CPU cache line in bytes. We want our lock structures
    672 	 *	aligned to this size to avoid false cache line sharing in the
    673 	 *	lock table.
    674 	 *	This value works for most CPUs. For Itanium this should be 128.
    675 	 */
    676 #ifndef CACHELINE
    677 #define CACHELINE	64
    678 #endif
    679 
    680 	/**	The information we store in a single slot of the reader table.
    681 	 *	In addition to a transaction ID, we also record the process and
    682 	 *	thread ID that owns a slot, so that we can detect stale information,
    683 	 *	e.g. threads or processes that went away without cleaning up.
    684 	 *	@note We currently don't check for stale records. We simply re-init
    685 	 *	the table when we know that we're the only process opening the
    686 	 *	lock file.
    687 	 */
    688 typedef struct MDB_rxbody {
    689 	/**	Current Transaction ID when this transaction began, or (txnid_t)-1.
    690 	 *	Multiple readers that start at the same time will probably have the
    691 	 *	same ID here. Again, it's not important to exclude them from
    692 	 *	anything; all we need to know is which version of the DB they
    693 	 *	started from so we can avoid overwriting any data used in that
    694 	 *	particular version.
    695 	 */
    696 	volatile txnid_t		mrb_txnid;
    697 	/** The process ID of the process owning this reader txn. */
    698 	volatile MDB_PID_T	mrb_pid;
    699 	/** The thread ID of the thread owning this txn. */
    700 	volatile MDB_THR_T	mrb_tid;
    701 } MDB_rxbody;
    702 
    703 	/** The actual reader record, with cacheline padding. */
    704 typedef struct MDB_reader {
    705 	union {
    706 		MDB_rxbody mrx;
    707 		/** shorthand for mrb_txnid */
    708 #define	mr_txnid	mru.mrx.mrb_txnid
    709 #define	mr_pid	mru.mrx.mrb_pid
    710 #define	mr_tid	mru.mrx.mrb_tid
    711 		/** cache line alignment */
    712 		char pad[(sizeof(MDB_rxbody)+CACHELINE-1) & ~(CACHELINE-1)];
    713 	} mru;
    714 } MDB_reader;
    715 
    716 	/** The header for the reader table.
    717 	 *	The table resides in a memory-mapped file. (This is a different file
    718 	 *	than is used for the main database.)
    719 	 *
    720 	 *	For POSIX the actual mutexes reside in the shared memory of this
    721 	 *	mapped file. On Windows, mutexes are named objects allocated by the
    722 	 *	kernel; we store the mutex names in this mapped file so that other
    723 	 *	processes can grab them. This same approach is also used on
    724 	 *	MacOSX/Darwin (using named semaphores) since MacOSX doesn't support
    725 	 *	process-shared POSIX mutexes. For these cases where a named object
    726 	 *	is used, the object name is derived from a 64 bit FNV hash of the
    727 	 *	environment pathname. As such, naming collisions are extremely
    728 	 *	unlikely. If a collision occurs, the results are unpredictable.
    729 	 */
    730 typedef struct MDB_txbody {
    731 		/** Stamp identifying this as an LMDB file. It must be set
    732 		 *	to #MDB_MAGIC. */
    733 	uint32_t	mtb_magic;
    734 		/** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */
    735 	uint32_t	mtb_format;
    736 #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
    737 	char	mtb_rmname[MNAME_LEN];
    738 #else
    739 		/** Mutex protecting access to this table.
    740 		 *	This is the reader table lock used with LOCK_MUTEX().
    741 		 */
    742 	mdb_mutex_t	mtb_rmutex;
    743 #endif
    744 		/**	The ID of the last transaction committed to the database.
    745 		 *	This is recorded here only for convenience; the value can always
    746 		 *	be determined by reading the main database meta pages.
    747 		 */
    748 	volatile txnid_t		mtb_txnid;
    749 		/** The number of slots that have been used in the reader table.
    750 		 *	This always records the maximum count, it is not decremented
    751 		 *	when readers release their slots.
    752 		 */
    753 	volatile unsigned	mtb_numreaders;
    754 } MDB_txbody;
    755 
    756 	/** The actual reader table definition. */
    757 typedef struct MDB_txninfo {
    758 	union {
    759 		MDB_txbody mtb;
    760 #define mti_magic	mt1.mtb.mtb_magic
    761 #define mti_format	mt1.mtb.mtb_format
    762 #define mti_rmutex	mt1.mtb.mtb_rmutex
    763 #define mti_rmname	mt1.mtb.mtb_rmname
    764 #define mti_txnid	mt1.mtb.mtb_txnid
    765 #define mti_numreaders	mt1.mtb.mtb_numreaders
    766 		char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)];
    767 	} mt1;
    768 	union {
    769 #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
    770 		char mt2_wmname[MNAME_LEN];
    771 #define	mti_wmname	mt2.mt2_wmname
    772 #else
    773 		mdb_mutex_t	mt2_wmutex;
    774 #define mti_wmutex	mt2.mt2_wmutex
    775 #endif
    776 		char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)];
    777 	} mt2;
    778 	MDB_reader	mti_readers[1];
    779 } MDB_txninfo;
    780 
    781 	/** Lockfile format signature: version, features and field layout */
    782 #define MDB_LOCK_FORMAT \
    783 	((uint32_t) \
    784 	 ((MDB_LOCK_VERSION) \
    785 	  /* Flags which describe functionality */ \
    786 	  + (((MDB_PIDLOCK) != 0) << 16)))
    787 /** @} */
    788 
    789 /** Common header for all page types. The page type depends on #mp_flags.
    790  *
    791  * #P_BRANCH and #P_LEAF pages have unsorted '#MDB_node's at the end, with
    792  * sorted #mp_ptrs[] entries referring to them. Exception: #P_LEAF2 pages
    793  * omit mp_ptrs and pack sorted #MDB_DUPFIXED values after the page header.
    794  *
    795  * #P_OVERFLOW records occupy one or more contiguous pages where only the
    796  * first has a page header. They hold the real data of #F_BIGDATA nodes.
    797  *
    798  * #P_SUBP sub-pages are small leaf "pages" with duplicate data.
    799  * A node with flag #F_DUPDATA but not #F_SUBDATA contains a sub-page.
    800  * (Duplicate data can also go in sub-databases, which use normal pages.)
    801  *
    802  * #P_META pages contain #MDB_meta, the start point of an LMDB snapshot.
    803  *
    804  * Each non-metapage up to #MDB_meta.%mm_last_pg is reachable exactly once
    805  * in the snapshot: Either used by a database or listed in a freeDB record.
    806  */
    807 typedef struct MDB_page {
    808 #define	mp_pgno	mp_p.p_pgno
    809 #define	mp_next	mp_p.p_next
    810 	union {
    811 		pgno_t		p_pgno;	/**< page number */
    812 		struct MDB_page *p_next; /**< for in-memory list of freed pages */
    813 	} mp_p;
    814 	uint16_t	mp_pad;			/**< key size if this is a LEAF2 page */
    815 /**	@defgroup mdb_page	Page Flags
    816  *	@ingroup internal
    817  *	Flags for the page headers.
    818  *	@{
    819  */
    820 #define	P_BRANCH	 0x01		/**< branch page */
    821 #define	P_LEAF		 0x02		/**< leaf page */
    822 #define	P_OVERFLOW	 0x04		/**< overflow page */
    823 #define	P_META		 0x08		/**< meta page */
    824 #define	P_DIRTY		 0x10		/**< dirty page, also set for #P_SUBP pages */
    825 #define	P_LEAF2		 0x20		/**< for #MDB_DUPFIXED records */
    826 #define	P_SUBP		 0x40		/**< for #MDB_DUPSORT sub-pages */
    827 #define	P_LOOSE		 0x4000		/**< page was dirtied then freed, can be reused */
    828 #define	P_KEEP		 0x8000		/**< leave this page alone during spill */
    829 /** @} */
    830 	uint16_t	mp_flags;		/**< @ref mdb_page */
    831 #define mp_lower	mp_pb.pb.pb_lower
    832 #define mp_upper	mp_pb.pb.pb_upper
    833 #define mp_pages	mp_pb.pb_pages
    834 	union {
    835 		struct {
    836 			indx_t		pb_lower;		/**< lower bound of free space */
    837 			indx_t		pb_upper;		/**< upper bound of free space */
    838 		} pb;
    839 		uint32_t	pb_pages;	/**< number of overflow pages */
    840 	} mp_pb;
    841 	indx_t		mp_ptrs[0];		/**< dynamic size */
    842 } MDB_page;
    843 
    844 /** Alternate page header, for 2-byte aligned access */
    845 typedef struct MDB_page2 {
    846 	uint16_t	mp2_p[sizeof(pgno_t)/2];
    847 	uint16_t	mp2_pad;
    848 	uint16_t	mp2_flags;
    849 	indx_t		mp2_lower;
    850 	indx_t		mp2_upper;
    851 	indx_t		mp2_ptrs[0];
    852 } MDB_page2;
    853 
    854 #define MP_PGNO(p)	(((MDB_page2 *)(void *)(p))->mp2_p)
    855 #define MP_PAD(p)	(((MDB_page2 *)(void *)(p))->mp2_pad)
    856 #define MP_FLAGS(p)	(((MDB_page2 *)(void *)(p))->mp2_flags)
    857 #define MP_LOWER(p)	(((MDB_page2 *)(void *)(p))->mp2_lower)
    858 #define MP_UPPER(p)	(((MDB_page2 *)(void *)(p))->mp2_upper)
    859 #define MP_PTRS(p)	(((MDB_page2 *)(void *)(p))->mp2_ptrs)
    860 
    861 	/** Size of the page header, excluding dynamic data at the end */
    862 #define PAGEHDRSZ	 ((unsigned) offsetof(MDB_page, mp_ptrs))
    863 
    864 	/** Address of first usable data byte in a page, after the header */
    865 #define METADATA(p)	 ((void *)((char *)(p) + PAGEHDRSZ))
    866 
    867 	/** ITS#7713, change PAGEBASE to handle 65536 byte pages */
    868 #define	PAGEBASE	((MDB_DEVEL) ? PAGEHDRSZ : 0)
    869 
    870 	/** Number of nodes on a page */
    871 #define NUMKEYS(p)	 ((MP_LOWER(p) - (PAGEHDRSZ-PAGEBASE)) >> 1)
    872 
    873 	/** The amount of space remaining in the page */
    874 #define SIZELEFT(p)	 (indx_t)(MP_UPPER(p) - MP_LOWER(p))
    875 
    876 	/** The percentage of space used in the page, in tenths of a percent. */
    877 #define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \
    878 				((env)->me_psize - PAGEHDRSZ))
    879 	/** The minimum page fill factor, in tenths of a percent.
    880 	 *	Pages emptier than this are candidates for merging.
    881 	 */
    882 #define FILL_THRESHOLD	 250
    883 
    884 	/** Test if a page is a leaf page */
    885 #define IS_LEAF(p)	 F_ISSET(MP_FLAGS(p), P_LEAF)
    886 	/** Test if a page is a LEAF2 page */
    887 #define IS_LEAF2(p)	 F_ISSET(MP_FLAGS(p), P_LEAF2)
    888 	/** Test if a page is a branch page */
    889 #define IS_BRANCH(p)	 F_ISSET(MP_FLAGS(p), P_BRANCH)
    890 	/** Test if a page is an overflow page */
    891 #define IS_OVERFLOW(p)	 F_ISSET(MP_FLAGS(p), P_OVERFLOW)
    892 	/** Test if a page is a sub page */
    893 #define IS_SUBP(p)	 F_ISSET(MP_FLAGS(p), P_SUBP)
    894 
    895 	/** The number of overflow pages needed to store the given size. */
    896 #define OVPAGES(size, psize)	((PAGEHDRSZ-1 + (size)) / (psize) + 1)
    897 
    898 	/** Link in #MDB_txn.%mt_loose_pgs list.
    899 	 *  Kept outside the page header, which is needed when reusing the page.
    900 	 */
    901 #define NEXT_LOOSE_PAGE(p)		(*(MDB_page **)((p) + 2))
    902 
    903 	/** Header for a single key/data pair within a page.
    904 	 * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2.
    905 	 * We guarantee 2-byte alignment for 'MDB_node's.
    906 	 *
    907 	 * #mn_lo and #mn_hi are used for data size on leaf nodes, and for child
    908 	 * pgno on branch nodes.  On 64 bit platforms, #mn_flags is also used
    909 	 * for pgno.  (Branch nodes have no flags).  Lo and hi are in host byte
    910 	 * order in case some accesses can be optimized to 32-bit word access.
    911 	 *
    912 	 * Leaf node flags describe node contents.  #F_BIGDATA says the node's
    913 	 * data part is the page number of an overflow page with actual data.
    914 	 * #F_DUPDATA and #F_SUBDATA can be combined giving duplicate data in
    915 	 * a sub-page/sub-database, and named databases (just #F_SUBDATA).
    916 	 */
    917 typedef struct MDB_node {
    918 	/** part of data size or pgno
    919 	 *	@{ */
    920 #if BYTE_ORDER == LITTLE_ENDIAN
    921 	unsigned short	mn_lo, mn_hi;
    922 #else
    923 	unsigned short	mn_hi, mn_lo;
    924 #endif
    925 	/** @} */
    926 /** @defgroup mdb_node Node Flags
    927  *	@ingroup internal
    928  *	Flags for node headers.
    929  *	@{
    930  */
    931 #define F_BIGDATA	 0x01			/**< data put on overflow page */
    932 #define F_SUBDATA	 0x02			/**< data is a sub-database */
    933 #define F_DUPDATA	 0x04			/**< data has duplicates */
    934 
    935 /** valid flags for #mdb_node_add() */
    936 #define	NODE_ADD_FLAGS	(F_DUPDATA|F_SUBDATA|MDB_RESERVE|MDB_APPEND)
    937 
    938 /** @} */
    939 	unsigned short	mn_flags;		/**< @ref mdb_node */
    940 	unsigned short	mn_ksize;		/**< key size */
    941 	char		mn_data[1];			/**< key and data are appended here */
    942 } MDB_node;
    943 
    944 	/** Size of the node header, excluding dynamic data at the end */
    945 #define NODESIZE	 offsetof(MDB_node, mn_data)
    946 
    947 	/** Bit position of top word in page number, for shifting mn_flags */
    948 #define PGNO_TOPWORD ((pgno_t)-1 > 0xffffffffu ? 32 : 0)
    949 
    950 	/** Size of a node in a branch page with a given key.
    951 	 *	This is just the node header plus the key, there is no data.
    952 	 */
    953 #define INDXSIZE(k)	 (NODESIZE + ((k) == NULL ? 0 : (k)->mv_size))
    954 
    955 	/** Size of a node in a leaf page with a given key and data.
    956 	 *	This is node header plus key plus data size.
    957 	 */
    958 #define LEAFSIZE(k, d)	 (NODESIZE + (k)->mv_size + (d)->mv_size)
    959 
    960 	/** Address of node \b i in page \b p */
    961 #define NODEPTR(p, i)	 ((MDB_node *)((char *)(p) + MP_PTRS(p)[i] + PAGEBASE))
    962 
    963 	/** Address of the key for the node */
    964 #define NODEKEY(node)	 (void *)((node)->mn_data)
    965 
    966 	/** Address of the data for a node */
    967 #define NODEDATA(node)	 (void *)((char *)(node)->mn_data + (node)->mn_ksize)
    968 
    969 	/** Get the page number pointed to by a branch node */
    970 #define NODEPGNO(node) \
    971 	((node)->mn_lo | ((pgno_t) (node)->mn_hi << 16) | \
    972 	 (PGNO_TOPWORD ? ((pgno_t) (node)->mn_flags << PGNO_TOPWORD) : 0))
    973 	/** Set the page number in a branch node */
    974 #define SETPGNO(node,pgno)	do { \
    975 	(node)->mn_lo = (pgno) & 0xffff; (node)->mn_hi = (pgno) >> 16; \
    976 	if (PGNO_TOPWORD) (node)->mn_flags = (pgno) >> PGNO_TOPWORD; } while(0)
    977 
    978 	/** Get the size of the data in a leaf node */
    979 #define NODEDSZ(node)	 ((node)->mn_lo | ((unsigned)(node)->mn_hi << 16))
    980 	/** Set the size of the data for a leaf node */
    981 #define SETDSZ(node,size)	do { \
    982 	(node)->mn_lo = (size) & 0xffff; (node)->mn_hi = (size) >> 16;} while(0)
    983 	/** The size of a key in a node */
    984 #define NODEKSZ(node)	 ((node)->mn_ksize)
    985 
    986 	/** Copy a page number from src to dst */
    987 #ifdef MISALIGNED_OK
    988 #define COPY_PGNO(dst,src)	dst = src
    989 #undef MP_PGNO
    990 #define MP_PGNO(p)	((p)->mp_pgno)
    991 #else
    992 #if SIZE_MAX > 4294967295UL
    993 #define COPY_PGNO(dst,src)	do { \
    994 	unsigned short *s, *d;	\
    995 	s = (unsigned short *)&(src);	\
    996 	d = (unsigned short *)&(dst);	\
    997 	*d++ = *s++;	\
    998 	*d++ = *s++;	\
    999 	*d++ = *s++;	\
   1000 	*d = *s;	\
   1001 } while (0)
   1002 #else
   1003 #define COPY_PGNO(dst,src)	do { \
   1004 	unsigned short *s, *d;	\
   1005 	s = (unsigned short *)&(src);	\
   1006 	d = (unsigned short *)&(dst);	\
   1007 	*d++ = *s++;	\
   1008 	*d = *s;	\
   1009 } while (0)
   1010 #endif
   1011 #endif
   1012 	/** The address of a key in a LEAF2 page.
   1013 	 *	LEAF2 pages are used for #MDB_DUPFIXED sorted-duplicate sub-DBs.
   1014 	 *	There are no node headers, keys are stored contiguously.
   1015 	 */
   1016 #define LEAF2KEY(p, i, ks)	((char *)(p) + PAGEHDRSZ + ((i)*(ks)))
   1017 
   1018 	/** Set the \b node's key into \b keyptr, if requested. */
   1019 #define MDB_GET_KEY(node, keyptr)	{ if ((keyptr) != NULL) { \
   1020 	(keyptr)->mv_size = NODEKSZ(node); (keyptr)->mv_data = NODEKEY(node); } }
   1021 
   1022 	/** Set the \b node's key into \b key. */
   1023 #define MDB_GET_KEY2(node, key)	{ key.mv_size = NODEKSZ(node); key.mv_data = NODEKEY(node); }
   1024 
   1025 	/** Information about a single database in the environment. */
   1026 typedef struct MDB_db {
   1027 	uint32_t	md_pad;		/**< also ksize for LEAF2 pages */
   1028 	uint16_t	md_flags;	/**< @ref mdb_dbi_open */
   1029 	uint16_t	md_depth;	/**< depth of this tree */
   1030 	pgno_t		md_branch_pages;	/**< number of internal pages */
   1031 	pgno_t		md_leaf_pages;		/**< number of leaf pages */
   1032 	pgno_t		md_overflow_pages;	/**< number of overflow pages */
   1033 	size_t		md_entries;		/**< number of data items */
   1034 	pgno_t		md_root;		/**< the root page of this tree */
   1035 } MDB_db;
   1036 
   1037 #define MDB_VALID	0x8000		/**< DB handle is valid, for me_dbflags */
   1038 #define PERSISTENT_FLAGS	(0xffff & ~(MDB_VALID))
   1039 	/** #mdb_dbi_open() flags */
   1040 #define VALID_FLAGS	(MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\
   1041 	MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE)
   1042 
   1043 	/** Handle for the DB used to track free pages. */
   1044 #define	FREE_DBI	0
   1045 	/** Handle for the default DB. */
   1046 #define	MAIN_DBI	1
   1047 	/** Number of DBs in metapage (free and main) - also hardcoded elsewhere */
   1048 #define CORE_DBS	2
   1049 
   1050 	/** Number of meta pages - also hardcoded elsewhere */
   1051 #define NUM_METAS	2
   1052 
   1053 	/** Meta page content.
   1054 	 *	A meta page is the start point for accessing a database snapshot.
   1055 	 *	Pages 0-1 are meta pages. Transaction N writes meta page #(N % 2).
   1056 	 */
   1057 typedef struct MDB_meta {
   1058 		/** Stamp identifying this as an LMDB file. It must be set
   1059 		 *	to #MDB_MAGIC. */
   1060 	uint32_t	mm_magic;
   1061 		/** Version number of this file. Must be set to #MDB_DATA_VERSION. */
   1062 	uint32_t	mm_version;
   1063 	void		*mm_address;		/**< address for fixed mapping */
   1064 	size_t		mm_mapsize;			/**< size of mmap region */
   1065 	MDB_db		mm_dbs[CORE_DBS];	/**< first is free space, 2nd is main db */
   1066 	/** The size of pages used in this DB */
   1067 #define	mm_psize	mm_dbs[FREE_DBI].md_pad
   1068 	/** Any persistent environment flags. @ref mdb_env */
   1069 #define	mm_flags	mm_dbs[FREE_DBI].md_flags
   1070 	/** Last used page in the datafile.
   1071 	 *	Actually the file may be shorter if the freeDB lists the final pages.
   1072 	 */
   1073 	pgno_t		mm_last_pg;
   1074 	volatile txnid_t	mm_txnid;	/**< txnid that committed this page */
   1075 } MDB_meta;
   1076 
   1077 	/** Buffer for a stack-allocated meta page.
   1078 	 *	The members define size and alignment, and silence type
   1079 	 *	aliasing warnings.  They are not used directly; that could
   1080 	 *	mean incorrectly using several union members in parallel.
   1081 	 */
   1082 typedef union MDB_metabuf {
   1083 	MDB_page	mb_page;
   1084 	struct {
   1085 		char		mm_pad[PAGEHDRSZ];
   1086 		MDB_meta	mm_meta;
   1087 	} mb_metabuf;
   1088 } MDB_metabuf;
   1089 
   1090 	/** Auxiliary DB info.
   1091 	 *	The information here is mostly static/read-only. There is
   1092 	 *	only a single copy of this record in the environment.
   1093 	 */
   1094 typedef struct MDB_dbx {
   1095 	MDB_val		md_name;		/**< name of the database */
   1096 	MDB_cmp_func	*md_cmp;	/**< function for comparing keys */
   1097 	MDB_cmp_func	*md_dcmp;	/**< function for comparing data items */
   1098 	MDB_rel_func	*md_rel;	/**< user relocate function */
   1099 	void		*md_relctx;		/**< user-provided context for md_rel */
   1100 } MDB_dbx;
   1101 
   1102 	/** A database transaction.
   1103 	 *	Every operation requires a transaction handle.
   1104 	 */
   1105 struct MDB_txn {
   1106 	MDB_txn		*mt_parent;		/**< parent of a nested txn */
   1107 	/** Nested txn under this txn, set together with flag #MDB_TXN_HAS_CHILD */
   1108 	MDB_txn		*mt_child;
   1109 	pgno_t		mt_next_pgno;	/**< next unallocated page */
   1110 	/** The ID of this transaction. IDs are integers incrementing from 1.
   1111 	 *	Only committed write transactions increment the ID. If a transaction
   1112 	 *	aborts, the ID may be re-used by the next writer.
   1113 	 */
   1114 	txnid_t		mt_txnid;
   1115 	MDB_env		*mt_env;		/**< the DB environment */
   1116 	/** The list of pages that became unused during this transaction.
   1117 	 */
   1118 	MDB_IDL		mt_free_pgs;
   1119 	/** The list of loose pages that became unused and may be reused
   1120 	 *	in this transaction, linked through #NEXT_LOOSE_PAGE(page).
   1121 	 */
   1122 	MDB_page	*mt_loose_pgs;
   1123 	/** Number of loose pages (#mt_loose_pgs) */
   1124 	int			mt_loose_count;
   1125 	/** The sorted list of dirty pages we temporarily wrote to disk
   1126 	 *	because the dirty list was full. page numbers in here are
   1127 	 *	shifted left by 1, deleted slots have the LSB set.
   1128 	 */
   1129 	MDB_IDL		mt_spill_pgs;
   1130 	union {
   1131 		/** For write txns: Modified pages. Sorted when not MDB_WRITEMAP. */
   1132 		MDB_ID2L	dirty_list;
   1133 		/** For read txns: This thread/txn's reader table slot, or NULL. */
   1134 		MDB_reader	*reader;
   1135 	} mt_u;
   1136 	/** Array of records for each DB known in the environment. */
   1137 	MDB_dbx		*mt_dbxs;
   1138 	/** Array of MDB_db records for each known DB */
   1139 	MDB_db		*mt_dbs;
   1140 	/** Array of sequence numbers for each DB handle */
   1141 	unsigned int	*mt_dbiseqs;
   1142 /** @defgroup mt_dbflag	Transaction DB Flags
   1143  *	@ingroup internal
   1144  * @{
   1145  */
   1146 #define DB_DIRTY	0x01		/**< DB was written in this txn */
   1147 #define DB_STALE	0x02		/**< Named-DB record is older than txnID */
   1148 #define DB_NEW		0x04		/**< Named-DB handle opened in this txn */
   1149 #define DB_VALID	0x08		/**< DB handle is valid, see also #MDB_VALID */
   1150 #define DB_USRVALID	0x10		/**< As #DB_VALID, but not set for #FREE_DBI */
   1151 #define DB_DUPDATA	0x20		/**< DB is #MDB_DUPSORT data */
   1152 /** @} */
   1153 	/** In write txns, array of cursors for each DB */
   1154 	MDB_cursor	**mt_cursors;
   1155 	/** Array of flags for each DB */
   1156 	unsigned char	*mt_dbflags;
   1157 	/**	Number of DB records in use, or 0 when the txn is finished.
   1158 	 *	This number only ever increments until the txn finishes; we
   1159 	 *	don't decrement it when individual DB handles are closed.
   1160 	 */
   1161 	MDB_dbi		mt_numdbs;
   1162 
   1163 /** @defgroup mdb_txn	Transaction Flags
   1164  *	@ingroup internal
   1165  *	@{
   1166  */
   1167 	/** #mdb_txn_begin() flags */
   1168 #define MDB_TXN_BEGIN_FLAGS	MDB_RDONLY
   1169 #define MDB_TXN_RDONLY		MDB_RDONLY	/**< read-only transaction */
   1170 	/* internal txn flags */
   1171 #define MDB_TXN_WRITEMAP	MDB_WRITEMAP	/**< copy of #MDB_env flag in writers */
   1172 #define MDB_TXN_FINISHED	0x01		/**< txn is finished or never began */
   1173 #define MDB_TXN_ERROR		0x02		/**< txn is unusable after an error */
   1174 #define MDB_TXN_DIRTY		0x04		/**< must write, even if dirty list is empty */
   1175 #define MDB_TXN_SPILLS		0x08		/**< txn or a parent has spilled pages */
   1176 #define MDB_TXN_HAS_CHILD	0x10		/**< txn has an #MDB_txn.%mt_child */
   1177 	/** most operations on the txn are currently illegal */
   1178 #define MDB_TXN_BLOCKED		(MDB_TXN_FINISHED|MDB_TXN_ERROR|MDB_TXN_HAS_CHILD)
   1179 /** @} */
   1180 	unsigned int	mt_flags;		/**< @ref mdb_txn */
   1181 	/** #dirty_list room: Array size - \#dirty pages visible to this txn.
   1182 	 *	Includes ancestor txns' dirty pages not hidden by other txns'
   1183 	 *	dirty/spilled pages. Thus commit(nested txn) has room to merge
   1184 	 *	dirty_list into mt_parent after freeing hidden mt_parent pages.
   1185 	 */
   1186 	unsigned int	mt_dirty_room;
   1187 };
   1188 
   1189 /** Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty.
   1190  * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to
   1191  * raise this on a 64 bit machine.
   1192  */
   1193 #define CURSOR_STACK		 32
   1194 
   1195 struct MDB_xcursor;
   1196 
   1197 	/** Cursors are used for all DB operations.
   1198 	 *	A cursor holds a path of (page pointer, key index) from the DB
   1199 	 *	root to a position in the DB, plus other state. #MDB_DUPSORT
   1200 	 *	cursors include an xcursor to the current data item. Write txns
   1201 	 *	track their cursors and keep them up to date when data moves.
   1202 	 *	Exception: An xcursor's pointer to a #P_SUBP page can be stale.
   1203 	 *	(A node with #F_DUPDATA but no #F_SUBDATA contains a subpage).
   1204 	 */
   1205 struct MDB_cursor {
   1206 	/** Next cursor on this DB in this txn */
   1207 	MDB_cursor	*mc_next;
   1208 	/** Backup of the original cursor if this cursor is a shadow */
   1209 	MDB_cursor	*mc_backup;
   1210 	/** Context used for databases with #MDB_DUPSORT, otherwise NULL */
   1211 	struct MDB_xcursor	*mc_xcursor;
   1212 	/** The transaction that owns this cursor */
   1213 	MDB_txn		*mc_txn;
   1214 	/** The database handle this cursor operates on */
   1215 	MDB_dbi		mc_dbi;
   1216 	/** The database record for this cursor */
   1217 	MDB_db		*mc_db;
   1218 	/** The database auxiliary record for this cursor */
   1219 	MDB_dbx		*mc_dbx;
   1220 	/** The @ref mt_dbflag for this database */
   1221 	unsigned char	*mc_dbflag;
   1222 	unsigned short 	mc_snum;	/**< number of pushed pages */
   1223 	unsigned short	mc_top;		/**< index of top page, normally mc_snum-1 */
   1224 /** @defgroup mdb_cursor	Cursor Flags
   1225  *	@ingroup internal
   1226  *	Cursor state flags.
   1227  *	@{
   1228  */
   1229 #define C_INITIALIZED	0x01	/**< cursor has been initialized and is valid */
   1230 #define C_EOF	0x02			/**< No more data */
   1231 #define C_SUB	0x04			/**< Cursor is a sub-cursor */
   1232 #define C_DEL	0x08			/**< last op was a cursor_del */
   1233 #define C_UNTRACK	0x40		/**< Un-track cursor when closing */
   1234 /** @} */
   1235 	unsigned int	mc_flags;	/**< @ref mdb_cursor */
   1236 	MDB_page	*mc_pg[CURSOR_STACK];	/**< stack of pushed pages */
   1237 	indx_t		mc_ki[CURSOR_STACK];	/**< stack of page indices */
   1238 };
   1239 
   1240 	/** Context for sorted-dup records.
   1241 	 *	We could have gone to a fully recursive design, with arbitrarily
   1242 	 *	deep nesting of sub-databases. But for now we only handle these
   1243 	 *	levels - main DB, optional sub-DB, sorted-duplicate DB.
   1244 	 */
   1245 typedef struct MDB_xcursor {
   1246 	/** A sub-cursor for traversing the Dup DB */
   1247 	MDB_cursor mx_cursor;
   1248 	/** The database record for this Dup DB */
   1249 	MDB_db	mx_db;
   1250 	/**	The auxiliary DB record for this Dup DB */
   1251 	MDB_dbx	mx_dbx;
   1252 	/** The @ref mt_dbflag for this Dup DB */
   1253 	unsigned char mx_dbflag;
   1254 } MDB_xcursor;
   1255 
   1256 	/** Check if there is an inited xcursor */
   1257 #define XCURSOR_INITED(mc) \
   1258 	((mc)->mc_xcursor && ((mc)->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))
   1259 
   1260 	/** Update the xcursor's sub-page pointer, if any, in \b mc.  Needed
   1261 	 *	when the node which contains the sub-page may have moved.  Called
   1262 	 *	with leaf page \b mp = mc->mc_pg[\b top].
   1263 	 */
   1264 #define XCURSOR_REFRESH(mc, top, mp) do { \
   1265 	MDB_page *xr_pg = (mp); \
   1266 	MDB_node *xr_node; \
   1267 	if (!XCURSOR_INITED(mc) || (mc)->mc_ki[top] >= NUMKEYS(xr_pg)) break; \
   1268 	xr_node = NODEPTR(xr_pg, (mc)->mc_ki[top]); \
   1269 	if ((xr_node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) \
   1270 		(mc)->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(xr_node); \
   1271 } while (0)
   1272 
   1273 	/** State of FreeDB old pages, stored in the MDB_env */
   1274 typedef struct MDB_pgstate {
   1275 	pgno_t		*mf_pghead;	/**< Reclaimed freeDB pages, or NULL before use */
   1276 	txnid_t		mf_pglast;	/**< ID of last used record, or 0 if !mf_pghead */
   1277 } MDB_pgstate;
   1278 
   1279 	/** The database environment. */
   1280 struct MDB_env {
   1281 	HANDLE		me_fd;		/**< The main data file */
   1282 	HANDLE		me_lfd;		/**< The lock file */
   1283 	HANDLE		me_mfd;		/**< For writing and syncing the meta pages */
   1284 	/** Failed to update the meta page. Probably an I/O error. */
   1285 #define	MDB_FATAL_ERROR	0x80000000U
   1286 	/** Some fields are initialized. */
   1287 #define	MDB_ENV_ACTIVE	0x20000000U
   1288 	/** me_txkey is set */
   1289 #define	MDB_ENV_TXKEY	0x10000000U
   1290 	/** fdatasync is unreliable */
   1291 #define	MDB_FSYNCONLY	0x08000000U
   1292 	uint32_t 	me_flags;		/**< @ref mdb_env */
   1293 	unsigned int	me_psize;	/**< DB page size, inited from me_os_psize */
   1294 	unsigned int	me_os_psize;	/**< OS page size, from #GET_PAGESIZE */
   1295 	unsigned int	me_maxreaders;	/**< size of the reader table */
   1296 	/** Max #MDB_txninfo.%mti_numreaders of interest to #mdb_env_close() */
   1297 	volatile int	me_close_readers;
   1298 	MDB_dbi		me_numdbs;		/**< number of DBs opened */
   1299 	MDB_dbi		me_maxdbs;		/**< size of the DB table */
   1300 	MDB_PID_T	me_pid;		/**< process ID of this env */
   1301 	char		*me_path;		/**< path to the DB files */
   1302 	char		*me_map;		/**< the memory map of the data file */
   1303 	MDB_txninfo	*me_txns;		/**< the memory map of the lock file or NULL */
   1304 	MDB_meta	*me_metas[NUM_METAS];	/**< pointers to the two meta pages */
   1305 	void		*me_pbuf;		/**< scratch area for DUPSORT put() */
   1306 	MDB_txn		*me_txn;		/**< current write transaction */
   1307 	MDB_txn		*me_txn0;		/**< prealloc'd write transaction */
   1308 	size_t		me_mapsize;		/**< size of the data memory map */
   1309 	off_t		me_size;		/**< current file size */
   1310 	pgno_t		me_maxpg;		/**< me_mapsize / me_psize */
   1311 	MDB_dbx		*me_dbxs;		/**< array of static DB info */
   1312 	uint16_t	*me_dbflags;	/**< array of flags from MDB_db.md_flags */
   1313 	unsigned int	*me_dbiseqs;	/**< array of dbi sequence numbers */
   1314 	pthread_key_t	me_txkey;	/**< thread-key for readers */
   1315 	txnid_t		me_pgoldest;	/**< ID of oldest reader last time we looked */
   1316 	MDB_pgstate	me_pgstate;		/**< state of old pages from freeDB */
   1317 #	define		me_pglast	me_pgstate.mf_pglast
   1318 #	define		me_pghead	me_pgstate.mf_pghead
   1319 	MDB_page	*me_dpages;		/**< list of malloc'd blocks for re-use */
   1320 	/** IDL of pages that became unused in a write txn */
   1321 	MDB_IDL		me_free_pgs;
   1322 	/** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. */
   1323 	MDB_ID2L	me_dirty_list;
   1324 	/** Max number of freelist items that can fit in a single overflow page */
   1325 	int			me_maxfree_1pg;
   1326 	/** Max size of a node on a page */
   1327 	unsigned int	me_nodemax;
   1328 #if !(MDB_MAXKEYSIZE)
   1329 	unsigned int	me_maxkey;	/**< max size of a key */
   1330 #endif
   1331 	int		me_live_reader;		/**< have liveness lock in reader table */
   1332 #ifdef _WIN32
   1333 	int		me_pidquery;		/**< Used in OpenProcess */
   1334 #endif
   1335 #ifdef MDB_USE_POSIX_MUTEX	/* Posix mutexes reside in shared mem */
   1336 #	define		me_rmutex	me_txns->mti_rmutex /**< Shared reader lock */
   1337 #	define		me_wmutex	me_txns->mti_wmutex /**< Shared writer lock */
   1338 #else
   1339 	mdb_mutex_t	me_rmutex;
   1340 	mdb_mutex_t	me_wmutex;
   1341 #endif
   1342 	void		*me_userctx;	 /**< User-settable context */
   1343 	MDB_assert_func *me_assert_func; /**< Callback for assertion failures */
   1344 };
   1345 
   1346 	/** Nested transaction */
   1347 typedef struct MDB_ntxn {
   1348 	MDB_txn		mnt_txn;		/**< the transaction */
   1349 	MDB_pgstate	mnt_pgstate;	/**< parent transaction's saved freestate */
   1350 } MDB_ntxn;
   1351 
   1352 	/** max number of pages to commit in one writev() call */
   1353 #define MDB_COMMIT_PAGES	 64
   1354 #if defined(IOV_MAX) && IOV_MAX < MDB_COMMIT_PAGES
   1355 #undef MDB_COMMIT_PAGES
   1356 #define MDB_COMMIT_PAGES	IOV_MAX
   1357 #endif
   1358 
   1359 	/** max bytes to write in one call */
   1360 #define MAX_WRITE		(0x40000000U >> (sizeof(ssize_t) == 4))
   1361 
   1362 	/** Check \b txn and \b dbi arguments to a function */
   1363 #define TXN_DBI_EXIST(txn, dbi, validity) \
   1364 	((txn) && (dbi)<(txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & (validity)))
   1365 
   1366 	/** Check for misused \b dbi handles */
   1367 #define TXN_DBI_CHANGED(txn, dbi) \
   1368 	((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi])
   1369 
   1370 static int  mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp);
   1371 static int  mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp);
   1372 static int  mdb_page_touch(MDB_cursor *mc);
   1373 
   1374 #define MDB_END_NAMES {"committed", "empty-commit", "abort", "reset", \
   1375 	"reset-tmp", "fail-begin", "fail-beginchild"}
   1376 enum {
   1377 	/* mdb_txn_end operation number, for logging */
   1378 	MDB_END_COMMITTED, MDB_END_EMPTY_COMMIT, MDB_END_ABORT, MDB_END_RESET,
   1379 	MDB_END_RESET_TMP, MDB_END_FAIL_BEGIN, MDB_END_FAIL_BEGINCHILD
   1380 };
   1381 #define MDB_END_OPMASK	0x0F	/**< mask for #mdb_txn_end() operation number */
   1382 #define MDB_END_UPDATE	0x10	/**< update env state (DBIs) */
   1383 #define MDB_END_FREE	0x20	/**< free txn unless it is #MDB_env.%me_txn0 */
   1384 #define MDB_END_SLOT MDB_NOTLS	/**< release any reader slot if #MDB_NOTLS */
   1385 static void mdb_txn_end(MDB_txn *txn, unsigned mode);
   1386 
   1387 static int  mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **mp, int *lvl);
   1388 static int  mdb_page_search_root(MDB_cursor *mc,
   1389 			    MDB_val *key, int modify);
   1390 #define MDB_PS_MODIFY	1
   1391 #define MDB_PS_ROOTONLY	2
   1392 #define MDB_PS_FIRST	4
   1393 #define MDB_PS_LAST		8
   1394 static int  mdb_page_search(MDB_cursor *mc,
   1395 			    MDB_val *key, int flags);
   1396 static int	mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst);
   1397 
   1398 #define MDB_SPLIT_REPLACE	MDB_APPENDDUP	/**< newkey is not new */
   1399 static int	mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata,
   1400 				pgno_t newpgno, unsigned int nflags);
   1401 
   1402 static int  mdb_env_read_header(MDB_env *env, MDB_meta *meta);
   1403 static MDB_meta *mdb_env_pick_meta(const MDB_env *env);
   1404 static int  mdb_env_write_meta(MDB_txn *txn);
   1405 #if defined(MDB_USE_POSIX_MUTEX) && !defined(MDB_ROBUST_SUPPORTED) /* Drop unused excl arg */
   1406 # define mdb_env_close0(env, excl) mdb_env_close1(env)
   1407 #endif
   1408 static void mdb_env_close0(MDB_env *env, int excl);
   1409 
   1410 static MDB_node *mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp);
   1411 static int  mdb_node_add(MDB_cursor *mc, indx_t indx,
   1412 			    MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags);
   1413 static void mdb_node_del(MDB_cursor *mc, int ksize);
   1414 static void mdb_node_shrink(MDB_page *mp, indx_t indx);
   1415 static int	mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft);
   1416 static int  mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data);
   1417 static size_t	mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data);
   1418 static size_t	mdb_branch_size(MDB_env *env, MDB_val *key);
   1419 
   1420 static int	mdb_rebalance(MDB_cursor *mc);
   1421 static int	mdb_update_key(MDB_cursor *mc, MDB_val *key);
   1422 
   1423 static void	mdb_cursor_pop(MDB_cursor *mc);
   1424 static int	mdb_cursor_push(MDB_cursor *mc, MDB_page *mp);
   1425 
   1426 static int	mdb_cursor_del0(MDB_cursor *mc);
   1427 static int	mdb_del0(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned flags);
   1428 static int	mdb_cursor_sibling(MDB_cursor *mc, int move_right);
   1429 static int	mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op);
   1430 static int	mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op);
   1431 static int	mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op,
   1432 				int *exactp);
   1433 static int	mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data);
   1434 static int	mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data);
   1435 
   1436 static void	mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx);
   1437 static void	mdb_xcursor_init0(MDB_cursor *mc);
   1438 static void	mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node);
   1439 static void	mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int force);
   1440 
   1441 static int	mdb_drop0(MDB_cursor *mc, int subs);
   1442 static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi);
   1443 static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead);
   1444 
   1445 /** @cond */
   1446 static MDB_cmp_func	mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long;
   1447 /** @endcond */
   1448 
   1449 /** Compare two items pointing at size_t's of unknown alignment. */
   1450 #ifdef MISALIGNED_OK
   1451 # define mdb_cmp_clong mdb_cmp_long
   1452 #else
   1453 # define mdb_cmp_clong mdb_cmp_cint
   1454 #endif
   1455 
   1456 #ifdef _WIN32
   1457 static SECURITY_DESCRIPTOR mdb_null_sd;
   1458 static SECURITY_ATTRIBUTES mdb_all_sa;
   1459 static int mdb_sec_inited;
   1460 
   1461 struct MDB_name;
   1462 static int utf8_to_utf16(const char *src, struct MDB_name *dst, int xtra);
   1463 #endif
   1464 
   1465 /** Return the library version info. */
   1466 char * ESECT
   1467 mdb_version(int *major, int *minor, int *patch)
   1468 {
   1469 	if (major) *major = MDB_VERSION_MAJOR;
   1470 	if (minor) *minor = MDB_VERSION_MINOR;
   1471 	if (patch) *patch = MDB_VERSION_PATCH;
   1472 	return MDB_VERSION_STRING;
   1473 }
   1474 
   1475 /** Table of descriptions for LMDB @ref errors */
   1476 static char *const mdb_errstr[] = {
   1477 	"MDB_KEYEXIST: Key/data pair already exists",
   1478 	"MDB_NOTFOUND: No matching key/data pair found",
   1479 	"MDB_PAGE_NOTFOUND: Requested page not found",
   1480 	"MDB_CORRUPTED: Located page was wrong type",
   1481 	"MDB_PANIC: Update of meta page failed or environment had fatal error",
   1482 	"MDB_VERSION_MISMATCH: Database environment version mismatch",
   1483 	"MDB_INVALID: File is not an LMDB file",
   1484 	"MDB_MAP_FULL: Environment mapsize limit reached",
   1485 	"MDB_DBS_FULL: Environment maxdbs limit reached",
   1486 	"MDB_READERS_FULL: Environment maxreaders limit reached",
   1487 	"MDB_TLS_FULL: Thread-local storage keys full - too many environments open",
   1488 	"MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big",
   1489 	"MDB_CURSOR_FULL: Internal error - cursor stack limit reached",
   1490 	"MDB_PAGE_FULL: Internal error - page has no more space",
   1491 	"MDB_MAP_RESIZED: Database contents grew beyond environment mapsize",
   1492 	"MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed",
   1493 	"MDB_BAD_RSLOT: Invalid reuse of reader locktable slot",
   1494 	"MDB_BAD_TXN: Transaction must abort, has a child, or is invalid",
   1495 	"MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size",
   1496 	"MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly",
   1497 };
   1498 
   1499 char *
   1500 mdb_strerror(int err)
   1501 {
   1502 #ifdef _WIN32
   1503 	/** HACK: pad 4KB on stack over the buf. Return system msgs in buf.
   1504 	 *	This works as long as no function between the call to mdb_strerror
   1505 	 *	and the actual use of the message uses more than 4K of stack.
   1506 	 */
   1507 #define MSGSIZE	1024
   1508 #define PADSIZE	4096
   1509 	char buf[MSGSIZE+PADSIZE], *ptr = buf;
   1510 #endif
   1511 	int i;
   1512 	if (!err)
   1513 		return ("Successful return: 0");
   1514 
   1515 	if (err >= MDB_KEYEXIST && err <= MDB_LAST_ERRCODE) {
   1516 		i = err - MDB_KEYEXIST;
   1517 		return mdb_errstr[i];
   1518 	}
   1519 
   1520 #ifdef _WIN32
   1521 	/* These are the C-runtime error codes we use. The comment indicates
   1522 	 * their numeric value, and the Win32 error they would correspond to
   1523 	 * if the error actually came from a Win32 API. A major mess, we should
   1524 	 * have used LMDB-specific error codes for everything.
   1525 	 */
   1526 	switch(err) {
   1527 	case ENOENT:	/* 2, FILE_NOT_FOUND */
   1528 	case EIO:		/* 5, ACCESS_DENIED */
   1529 	case ENOMEM:	/* 12, INVALID_ACCESS */
   1530 	case EACCES:	/* 13, INVALID_DATA */
   1531 	case EBUSY:		/* 16, CURRENT_DIRECTORY */
   1532 	case EINVAL:	/* 22, BAD_COMMAND */
   1533 	case ENOSPC:	/* 28, OUT_OF_PAPER */
   1534 		return strerror(err);
   1535 	default:
   1536 		;
   1537 	}
   1538 	buf[0] = 0;
   1539 	FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM |
   1540 		FORMAT_MESSAGE_IGNORE_INSERTS,
   1541 		NULL, err, 0, ptr, MSGSIZE, (va_list *)buf+MSGSIZE);
   1542 	return ptr;
   1543 #else
   1544 	if (err < 0)
   1545 		return "Invalid error code";
   1546 	return strerror(err);
   1547 #endif
   1548 }
   1549 
   1550 /** assert(3) variant in cursor context */
   1551 #define mdb_cassert(mc, expr)	mdb_assert0((mc)->mc_txn->mt_env, expr, #expr)
   1552 /** assert(3) variant in transaction context */
   1553 #define mdb_tassert(txn, expr)	mdb_assert0((txn)->mt_env, expr, #expr)
   1554 /** assert(3) variant in environment context */
   1555 #define mdb_eassert(env, expr)	mdb_assert0(env, expr, #expr)
   1556 
   1557 #ifndef NDEBUG
   1558 # define mdb_assert0(env, expr, expr_txt) ((expr) ? (void)0 : \
   1559 		mdb_assert_fail(env, expr_txt, mdb_func_, __FILE__, __LINE__))
   1560 
   1561 static void ESECT
   1562 mdb_assert_fail(MDB_env *env, const char *expr_txt,
   1563 	const char *func, const char *file, int line)
   1564 {
   1565 	char buf[400];
   1566 	sprintf(buf, "%.100s:%d: Assertion '%.200s' failed in %.40s()",
   1567 		file, line, expr_txt, func);
   1568 	if (env->me_assert_func)
   1569 		env->me_assert_func(env, buf);
   1570 	fprintf(stderr, "%s\n", buf);
   1571 	abort();
   1572 }
   1573 #else
   1574 # define mdb_assert0(env, expr, expr_txt) ((void) 0)
   1575 #endif /* NDEBUG */
   1576 
   1577 #if MDB_DEBUG
   1578 /** Return the page number of \b mp which may be sub-page, for debug output */
   1579 static pgno_t
   1580 mdb_dbg_pgno(MDB_page *mp)
   1581 {
   1582 	pgno_t ret;
   1583 	COPY_PGNO(ret, MP_PGNO(mp));
   1584 	return ret;
   1585 }
   1586 
   1587 /** Display a key in hexadecimal and return the address of the result.
   1588  * @param[in] key the key to display
   1589  * @param[in] buf the buffer to write into. Should always be #DKBUF.
   1590  * @return The key in hexadecimal form.
   1591  */
   1592 char *
   1593 mdb_dkey(MDB_val *key, char *buf)
   1594 {
   1595 	char *ptr = buf;
   1596 	unsigned char *c = key->mv_data;
   1597 	unsigned int i;
   1598 
   1599 	if (!key)
   1600 		return "";
   1601 
   1602 	if (key->mv_size > DKBUF_MAXKEYSIZE)
   1603 		return "MDB_MAXKEYSIZE";
   1604 	/* may want to make this a dynamic check: if the key is mostly
   1605 	 * printable characters, print it as-is instead of converting to hex.
   1606 	 */
   1607 #if 1
   1608 	buf[0] = '\0';
   1609 	for (i=0; i<key->mv_size; i++)
   1610 		ptr += sprintf(ptr, "%02x", *c++);
   1611 #else
   1612 	sprintf(buf, "%.*s", key->mv_size, key->mv_data);
   1613 #endif
   1614 	return buf;
   1615 }
   1616 
   1617 static const char *
   1618 mdb_leafnode_type(MDB_node *n)
   1619 {
   1620 	static char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}};
   1621 	return F_ISSET(n->mn_flags, F_BIGDATA) ? ": overflow page" :
   1622 		tp[F_ISSET(n->mn_flags, F_DUPDATA)][F_ISSET(n->mn_flags, F_SUBDATA)];
   1623 }
   1624 
   1625 /** Display all the keys in the page. */
   1626 void
   1627 mdb_page_list(MDB_page *mp)
   1628 {
   1629 	pgno_t pgno = mdb_dbg_pgno(mp);
   1630 	const char *type, *state = (MP_FLAGS(mp) & P_DIRTY) ? ", dirty" : "";
   1631 	MDB_node *node;
   1632 	unsigned int i, nkeys, nsize, total = 0;
   1633 	MDB_val key;
   1634 	DKBUF;
   1635 
   1636 	switch (MP_FLAGS(mp) & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) {
   1637 	case P_BRANCH:              type = "Branch page";		break;
   1638 	case P_LEAF:                type = "Leaf page";			break;
   1639 	case P_LEAF|P_SUBP:         type = "Sub-page";			break;
   1640 	case P_LEAF|P_LEAF2:        type = "LEAF2 page";		break;
   1641 	case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page";	break;
   1642 	case P_OVERFLOW:
   1643 		fprintf(stderr, "Overflow page %"Z"u pages %u%s\n",
   1644 			pgno, mp->mp_pages, state);
   1645 		return;
   1646 	case P_META:
   1647 		fprintf(stderr, "Meta-page %"Z"u txnid %"Z"u\n",
   1648 			pgno, ((MDB_meta *)METADATA(mp))->mm_txnid);
   1649 		return;
   1650 	default:
   1651 		fprintf(stderr, "Bad page %"Z"u flags 0x%X\n", pgno, MP_FLAGS(mp));
   1652 		return;
   1653 	}
   1654 
   1655 	nkeys = NUMKEYS(mp);
   1656 	fprintf(stderr, "%s %"Z"u numkeys %d%s\n", type, pgno, nkeys, state);
   1657 
   1658 	for (i=0; i<nkeys; i++) {
   1659 		if (IS_LEAF2(mp)) {	/* LEAF2 pages have no mp_ptrs[] or node headers */
   1660 			key.mv_size = nsize = mp->mp_pad;
   1661 			key.mv_data = LEAF2KEY(mp, i, nsize);
   1662 			total += nsize;
   1663 			fprintf(stderr, "key %d: nsize %d, %s\n", i, nsize, DKEY(&key));
   1664 			continue;
   1665 		}
   1666 		node = NODEPTR(mp, i);
   1667 		key.mv_size = node->mn_ksize;
   1668 		key.mv_data = node->mn_data;
   1669 		nsize = NODESIZE + key.mv_size;
   1670 		if (IS_BRANCH(mp)) {
   1671 			fprintf(stderr, "key %d: page %"Z"u, %s\n", i, NODEPGNO(node),
   1672 				DKEY(&key));
   1673 			total += nsize;
   1674 		} else {
   1675 			if (F_ISSET(node->mn_flags, F_BIGDATA))
   1676 				nsize += sizeof(pgno_t);
   1677 			else
   1678 				nsize += NODEDSZ(node);
   1679 			total += nsize;
   1680 			nsize += sizeof(indx_t);
   1681 			fprintf(stderr, "key %d: nsize %d, %s%s\n",
   1682 				i, nsize, DKEY(&key), mdb_leafnode_type(node));
   1683 		}
   1684 		total = EVEN(total);
   1685 	}
   1686 	fprintf(stderr, "Total: header %d + contents %d + unused %d\n",
   1687 		IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + MP_LOWER(mp), total, SIZELEFT(mp));
   1688 }
   1689 
   1690 void
   1691 mdb_cursor_chk(MDB_cursor *mc)
   1692 {
   1693 	unsigned int i;
   1694 	MDB_node *node;
   1695 	MDB_page *mp;
   1696 
   1697 	if (!mc->mc_snum || !(mc->mc_flags & C_INITIALIZED)) return;
   1698 	for (i=0; i<mc->mc_top; i++) {
   1699 		mp = mc->mc_pg[i];
   1700 		node = NODEPTR(mp, mc->mc_ki[i]);
   1701 		if (NODEPGNO(node) != mc->mc_pg[i+1]->mp_pgno)
   1702 			printf("oops!\n");
   1703 	}
   1704 	if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i]))
   1705 		printf("ack!\n");
   1706 	if (XCURSOR_INITED(mc)) {
   1707 		node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
   1708 		if (((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) &&
   1709 			mc->mc_xcursor->mx_cursor.mc_pg[0] != NODEDATA(node)) {
   1710 			printf("blah!\n");
   1711 		}
   1712 	}
   1713 }
   1714 #endif
   1715 
   1716 #if (MDB_DEBUG) > 2
   1717 /** Count all the pages in each DB and in the freelist
   1718  *  and make sure it matches the actual number of pages
   1719  *  being used.
   1720  *  All named DBs must be open for a correct count.
   1721  */
   1722 static void mdb_audit(MDB_txn *txn)
   1723 {
   1724 	MDB_cursor mc;
   1725 	MDB_val key, data;
   1726 	MDB_ID freecount, count;
   1727 	MDB_dbi i;
   1728 	int rc;
   1729 
   1730 	freecount = 0;
   1731 	mdb_cursor_init(&mc, txn, FREE_DBI, NULL);
   1732 	while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0)
   1733 		freecount += *(MDB_ID *)data.mv_data;
   1734 	mdb_tassert(txn, rc == MDB_NOTFOUND);
   1735 
   1736 	count = 0;
   1737 	for (i = 0; i<txn->mt_numdbs; i++) {
   1738 		MDB_xcursor mx;
   1739 		if (!(txn->mt_dbflags[i] & DB_VALID))
   1740 			continue;
   1741 		mdb_cursor_init(&mc, txn, i, &mx);
   1742 		if (txn->mt_dbs[i].md_root == P_INVALID)
   1743 			continue;
   1744 		count += txn->mt_dbs[i].md_branch_pages +
   1745 			txn->mt_dbs[i].md_leaf_pages +
   1746 			txn->mt_dbs[i].md_overflow_pages;
   1747 		if (txn->mt_dbs[i].md_flags & MDB_DUPSORT) {
   1748 			rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST);
   1749 			for (; rc == MDB_SUCCESS; rc = mdb_cursor_sibling(&mc, 1)) {
   1750 				unsigned j;
   1751 				MDB_page *mp;
   1752 				mp = mc.mc_pg[mc.mc_top];
   1753 				for (j=0; j<NUMKEYS(mp); j++) {
   1754 					MDB_node *leaf = NODEPTR(mp, j);
   1755 					if (leaf->mn_flags & F_SUBDATA) {
   1756 						MDB_db db;
   1757 						memcpy(&db, NODEDATA(leaf), sizeof(db));
   1758 						count += db.md_branch_pages + db.md_leaf_pages +
   1759 							db.md_overflow_pages;
   1760 					}
   1761 				}
   1762 			}
   1763 			mdb_tassert(txn, rc == MDB_NOTFOUND);
   1764 		}
   1765 	}
   1766 	if (freecount + count + NUM_METAS != txn->mt_next_pgno) {
   1767 		fprintf(stderr, "audit: %"Z"u freecount: %"Z"u count: %"Z"u total: %"Z"u next_pgno: %"Z"u\n",
   1768 			txn->mt_txnid, freecount, count+NUM_METAS,
   1769 			freecount+count+NUM_METAS, txn->mt_next_pgno);
   1770 	}
   1771 }
   1772 #endif
   1773 
   1774 int
   1775 mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
   1776 {
   1777 	return txn->mt_dbxs[dbi].md_cmp(a, b);
   1778 }
   1779 
   1780 int
   1781 mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
   1782 {
   1783 	MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp;
   1784 #if UINT_MAX < SIZE_MAX
   1785 	if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t))
   1786 		dcmp = mdb_cmp_clong;
   1787 #endif
   1788 	return dcmp(a, b);
   1789 }
   1790 
   1791 /** Allocate memory for a page.
   1792  * Re-use old malloc'd pages first for singletons, otherwise just malloc.
   1793  * Set #MDB_TXN_ERROR on failure.
   1794  */
   1795 static MDB_page *
   1796 mdb_page_malloc(MDB_txn *txn, unsigned num)
   1797 {
   1798 	MDB_env *env = txn->mt_env;
   1799 	MDB_page *ret = env->me_dpages;
   1800 	size_t psize = env->me_psize, sz = psize, off;
   1801 	/* For ! #MDB_NOMEMINIT, psize counts how much to init.
   1802 	 * For a single page alloc, we init everything after the page header.
   1803 	 * For multi-page, we init the final page; if the caller needed that
   1804 	 * many pages they will be filling in at least up to the last page.
   1805 	 */
   1806 	if (num == 1) {
   1807 		if (ret) {
   1808 			VGMEMP_ALLOC(env, ret, sz);
   1809 			VGMEMP_DEFINED(ret, sizeof(ret->mp_next));
   1810 			env->me_dpages = ret->mp_next;
   1811 			return ret;
   1812 		}
   1813 		psize -= off = PAGEHDRSZ;
   1814 	} else {
   1815 		sz *= num;
   1816 		off = sz - psize;
   1817 	}
   1818 	if ((ret = malloc(sz)) != NULL) {
   1819 		VGMEMP_ALLOC(env, ret, sz);
   1820 		if (!(env->me_flags & MDB_NOMEMINIT)) {
   1821 			memset((char *)ret + off, 0, psize);
   1822 			ret->mp_pad = 0;
   1823 		}
   1824 	} else {
   1825 		txn->mt_flags |= MDB_TXN_ERROR;
   1826 	}
   1827 	return ret;
   1828 }
   1829 /** Free a single page.
   1830  * Saves single pages to a list, for future reuse.
   1831  * (This is not used for multi-page overflow pages.)
   1832  */
   1833 static void
   1834 mdb_page_free(MDB_env *env, MDB_page *mp)
   1835 {
   1836 	mp->mp_next = env->me_dpages;
   1837 	VGMEMP_FREE(env, mp);
   1838 	env->me_dpages = mp;
   1839 }
   1840 
   1841 /** Free a dirty page */
   1842 static void
   1843 mdb_dpage_free(MDB_env *env, MDB_page *dp)
   1844 {
   1845 	if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) {
   1846 		mdb_page_free(env, dp);
   1847 	} else {
   1848 		/* large pages just get freed directly */
   1849 		VGMEMP_FREE(env, dp);
   1850 		free(dp);
   1851 	}
   1852 }
   1853 
   1854 /**	Return all dirty pages to dpage list */
   1855 static void
   1856 mdb_dlist_free(MDB_txn *txn)
   1857 {
   1858 	MDB_env *env = txn->mt_env;
   1859 	MDB_ID2L dl = txn->mt_u.dirty_list;
   1860 	unsigned i, n = dl[0].mid;
   1861 
   1862 	for (i = 1; i <= n; i++) {
   1863 		mdb_dpage_free(env, dl[i].mptr);
   1864 	}
   1865 	dl[0].mid = 0;
   1866 }
   1867 
   1868 /** Loosen or free a single page.
   1869  * Saves single pages to a list for future reuse
   1870  * in this same txn. It has been pulled from the freeDB
   1871  * and already resides on the dirty list, but has been
   1872  * deleted. Use these pages first before pulling again
   1873  * from the freeDB.
   1874  *
   1875  * If the page wasn't dirtied in this txn, just add it
   1876  * to this txn's free list.
   1877  */
   1878 static int
   1879 mdb_page_loose(MDB_cursor *mc, MDB_page *mp)
   1880 {
   1881 	int loose = 0;
   1882 	pgno_t pgno = mp->mp_pgno;
   1883 	MDB_txn *txn = mc->mc_txn;
   1884 
   1885 	if ((mp->mp_flags & P_DIRTY) && mc->mc_dbi != FREE_DBI) {
   1886 		if (txn->mt_parent) {
   1887 			MDB_ID2 *dl = txn->mt_u.dirty_list;
   1888 			/* If txn has a parent, make sure the page is in our
   1889 			 * dirty list.
   1890 			 */
   1891 			if (dl[0].mid) {
   1892 				unsigned x = mdb_mid2l_search(dl, pgno);
   1893 				if (x <= dl[0].mid && dl[x].mid == pgno) {
   1894 					if (mp != dl[x].mptr) { /* bad cursor? */
   1895 						mc->mc_flags &= ~(C_INITIALIZED|C_EOF);
   1896 						txn->mt_flags |= MDB_TXN_ERROR;
   1897 						return MDB_CORRUPTED;
   1898 					}
   1899 					/* ok, it's ours */
   1900 					loose = 1;
   1901 				}
   1902 			}
   1903 		} else {
   1904 			/* no parent txn, so it's just ours */
   1905 			loose = 1;
   1906 		}
   1907 	}
   1908 	if (loose) {
   1909 		DPRINTF(("loosen db %d page %"Z"u", DDBI(mc),
   1910 			mp->mp_pgno));
   1911 		NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs;
   1912 		txn->mt_loose_pgs = mp;
   1913 		txn->mt_loose_count++;
   1914 		mp->mp_flags |= P_LOOSE;
   1915 	} else {
   1916 		int rc = mdb_midl_append(&txn->mt_free_pgs, pgno);
   1917 		if (rc)
   1918 			return rc;
   1919 	}
   1920 
   1921 	return MDB_SUCCESS;
   1922 }
   1923 
   1924 /** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn.
   1925  * @param[in] mc A cursor handle for the current operation.
   1926  * @param[in] pflags Flags of the pages to update:
   1927  * P_DIRTY to set P_KEEP, P_DIRTY|P_KEEP to clear it.
   1928  * @param[in] all No shortcuts. Needed except after a full #mdb_page_flush().
   1929  * @return 0 on success, non-zero on failure.
   1930  */
   1931 static int
   1932 mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all)
   1933 {
   1934 	enum { Mask = P_SUBP|P_DIRTY|P_LOOSE|P_KEEP };
   1935 	MDB_txn *txn = mc->mc_txn;
   1936 	MDB_cursor *m3, *m0 = mc;
   1937 	MDB_xcursor *mx;
   1938 	MDB_page *dp, *mp;
   1939 	MDB_node *leaf;
   1940 	unsigned i, j;
   1941 	int rc = MDB_SUCCESS, level;
   1942 
   1943 	/* Mark pages seen by cursors */
   1944 	if (mc->mc_flags & C_UNTRACK)
   1945 		mc = NULL;				/* will find mc in mt_cursors */
   1946 	for (i = txn->mt_numdbs;; mc = txn->mt_cursors[--i]) {
   1947 		for (; mc; mc=mc->mc_next) {
   1948 			if (!(mc->mc_flags & C_INITIALIZED))
   1949 				continue;
   1950 			for (m3 = mc;; m3 = &mx->mx_cursor) {
   1951 				mp = NULL;
   1952 				for (j=0; j<m3->mc_snum; j++) {
   1953 					mp = m3->mc_pg[j];
   1954 					if ((mp->mp_flags & Mask) == pflags)
   1955 						mp->mp_flags ^= P_KEEP;
   1956 				}
   1957 				mx = m3->mc_xcursor;
   1958 				/* Proceed to mx if it is at a sub-database */
   1959 				if (! (mx && (mx->mx_cursor.mc_flags & C_INITIALIZED)))
   1960 					break;
   1961 				if (! (mp && (mp->mp_flags & P_LEAF)))
   1962 					break;
   1963 				leaf = NODEPTR(mp, m3->mc_ki[j-1]);
   1964 				if (!(leaf->mn_flags & F_SUBDATA))
   1965 					break;
   1966 			}
   1967 		}
   1968 		if (i == 0)
   1969 			break;
   1970 	}
   1971 
   1972 	if (all) {
   1973 		/* Mark dirty root pages */
   1974 		for (i=0; i<txn->mt_numdbs; i++) {
   1975 			if (txn->mt_dbflags[i] & DB_DIRTY) {
   1976 				pgno_t pgno = txn->mt_dbs[i].md_root;
   1977 				if (pgno == P_INVALID)
   1978 					continue;
   1979 				if ((rc = mdb_page_get(m0, pgno, &dp, &level)) != MDB_SUCCESS)
   1980 					break;
   1981 				if ((dp->mp_flags & Mask) == pflags && level <= 1)
   1982 					dp->mp_flags ^= P_KEEP;
   1983 			}
   1984 		}
   1985 	}
   1986 
   1987 	return rc;
   1988 }
   1989 
   1990 static int mdb_page_flush(MDB_txn *txn, int keep);
   1991 
   1992 /**	Spill pages from the dirty list back to disk.
   1993  * This is intended to prevent running into #MDB_TXN_FULL situations,
   1994  * but note that they may still occur in a few cases:
   1995  *	1) our estimate of the txn size could be too small. Currently this
   1996  *	 seems unlikely, except with a large number of #MDB_MULTIPLE items.
   1997  *	2) child txns may run out of space if their parents dirtied a
   1998  *	 lot of pages and never spilled them. TODO: we probably should do
   1999  *	 a preemptive spill during #mdb_txn_begin() of a child txn, if
   2000  *	 the parent's dirty_room is below a given threshold.
   2001  *
   2002  * Otherwise, if not using nested txns, it is expected that apps will
   2003  * not run into #MDB_TXN_FULL any more. The pages are flushed to disk
   2004  * the same way as for a txn commit, e.g. their P_DIRTY flag is cleared.
   2005  * If the txn never references them again, they can be left alone.
   2006  * If the txn only reads them, they can be used without any fuss.
   2007  * If the txn writes them again, they can be dirtied immediately without
   2008  * going thru all of the work of #mdb_page_touch(). Such references are
   2009  * handled by #mdb_page_unspill().
   2010  *
   2011  * Also note, we never spill DB root pages, nor pages of active cursors,
   2012  * because we'll need these back again soon anyway. And in nested txns,
   2013  * we can't spill a page in a child txn if it was already spilled in a
   2014  * parent txn. That would alter the parent txns' data even though
   2015  * the child hasn't committed yet, and we'd have no way to undo it if
   2016  * the child aborted.
   2017  *
   2018  * @param[in] m0 cursor A cursor handle identifying the transaction and
   2019  *	database for which we are checking space.
   2020  * @param[in] key For a put operation, the key being stored.
   2021  * @param[in] data For a put operation, the data being stored.
   2022  * @return 0 on success, non-zero on failure.
   2023  */
   2024 static int
   2025 mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data)
   2026 {
   2027 	MDB_txn *txn = m0->mc_txn;
   2028 	MDB_page *dp;
   2029 	MDB_ID2L dl = txn->mt_u.dirty_list;
   2030 	unsigned int i, j, need;
   2031 	int rc;
   2032 
   2033 	if (m0->mc_flags & C_SUB)
   2034 		return MDB_SUCCESS;
   2035 
   2036 	/* Estimate how much space this op will take */
   2037 	i = m0->mc_db->md_depth;
   2038 	/* Named DBs also dirty the main DB */
   2039 	if (m0->mc_dbi >= CORE_DBS)
   2040 		i += txn->mt_dbs[MAIN_DBI].md_depth;
   2041 	/* For puts, roughly factor in the key+data size */
   2042 	if (key)
   2043 		i += (LEAFSIZE(key, data) + txn->mt_env->me_psize) / txn->mt_env->me_psize;
   2044 	i += i;	/* double it for good measure */
   2045 	need = i;
   2046 
   2047 	if (txn->mt_dirty_room > i)
   2048 		return MDB_SUCCESS;
   2049 
   2050 	if (!txn->mt_spill_pgs) {
   2051 		txn->mt_spill_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX);
   2052 		if (!txn->mt_spill_pgs)
   2053 			return ENOMEM;
   2054 	} else {
   2055 		/* purge deleted slots */
   2056 		MDB_IDL sl = txn->mt_spill_pgs;
   2057 		unsigned int num = sl[0];
   2058 		j=0;
   2059 		for (i=1; i<=num; i++) {
   2060 			if (!(sl[i] & 1))
   2061 				sl[++j] = sl[i];
   2062 		}
   2063 		sl[0] = j;
   2064 	}
   2065 
   2066 	/* Preserve pages which may soon be dirtied again */
   2067 	if ((rc = mdb_pages_xkeep(m0, P_DIRTY, 1)) != MDB_SUCCESS)
   2068 		goto done;
   2069 
   2070 	/* Less aggressive spill - we originally spilled the entire dirty list,
   2071 	 * with a few exceptions for cursor pages and DB root pages. But this
   2072 	 * turns out to be a lot of wasted effort because in a large txn many
   2073 	 * of those pages will need to be used again. So now we spill only 1/8th
   2074 	 * of the dirty pages. Testing revealed this to be a good tradeoff,
   2075 	 * better than 1/2, 1/4, or 1/10.
   2076 	 */
   2077 	if (need < MDB_IDL_UM_MAX / 8)
   2078 		need = MDB_IDL_UM_MAX / 8;
   2079 
   2080 	/* Save the page IDs of all the pages we're flushing */
   2081 	/* flush from the tail forward, this saves a lot of shifting later on. */
   2082 	for (i=dl[0].mid; i && need; i--) {
   2083 		MDB_ID pn = dl[i].mid << 1;
   2084 		dp = dl[i].mptr;
   2085 		if (dp->mp_flags & (P_LOOSE|P_KEEP))
   2086 			continue;
   2087 		/* Can't spill twice, make sure it's not already in a parent's
   2088 		 * spill list.
   2089 		 */
   2090 		if (txn->mt_parent) {
   2091 			MDB_txn *tx2;
   2092 			for (tx2 = txn->mt_parent; tx2; tx2 = tx2->mt_parent) {
   2093 				if (tx2->mt_spill_pgs) {
   2094 					j = mdb_midl_search(tx2->mt_spill_pgs, pn);
   2095 					if (j <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[j] == pn) {
   2096 						dp->mp_flags |= P_KEEP;
   2097 						break;
   2098 					}
   2099 				}
   2100 			}
   2101 			if (tx2)
   2102 				continue;
   2103 		}
   2104 		if ((rc = mdb_midl_append(&txn->mt_spill_pgs, pn)))
   2105 			goto done;
   2106 		need--;
   2107 	}
   2108 	mdb_midl_sort(txn->mt_spill_pgs);
   2109 
   2110 	/* Flush the spilled part of dirty list */
   2111 	if ((rc = mdb_page_flush(txn, i)) != MDB_SUCCESS)
   2112 		goto done;
   2113 
   2114 	/* Reset any dirty pages we kept that page_flush didn't see */
   2115 	rc = mdb_pages_xkeep(m0, P_DIRTY|P_KEEP, i);
   2116 
   2117 done:
   2118 	txn->mt_flags |= rc ? MDB_TXN_ERROR : MDB_TXN_SPILLS;
   2119 	return rc;
   2120 }
   2121 
   2122 /** Find oldest txnid still referenced. Expects txn->mt_txnid > 0. */
   2123 static txnid_t
   2124 mdb_find_oldest(MDB_txn *txn)
   2125 {
   2126 	int i;
   2127 	txnid_t mr, oldest = txn->mt_txnid - 1;
   2128 	if (txn->mt_env->me_txns) {
   2129 		MDB_reader *r = txn->mt_env->me_txns->mti_readers;
   2130 		for (i = txn->mt_env->me_txns->mti_numreaders; --i >= 0; ) {
   2131 			if (r[i].mr_pid) {
   2132 				mr = r[i].mr_txnid;
   2133 				if (oldest > mr)
   2134 					oldest = mr;
   2135 			}
   2136 		}
   2137 	}
   2138 	return oldest;
   2139 }
   2140 
   2141 /** Add a page to the txn's dirty list */
   2142 static void
   2143 mdb_page_dirty(MDB_txn *txn, MDB_page *mp)
   2144 {
   2145 	MDB_ID2 mid;
   2146 	int rc, (*insert)(MDB_ID2L, MDB_ID2 *);
   2147 
   2148 	if (txn->mt_flags & MDB_TXN_WRITEMAP) {
   2149 		insert = mdb_mid2l_append;
   2150 	} else {
   2151 		insert = mdb_mid2l_insert;
   2152 	}
   2153 	mid.mid = mp->mp_pgno;
   2154 	mid.mptr = mp;
   2155 	rc = insert(txn->mt_u.dirty_list, &mid);
   2156 	mdb_tassert(txn, rc == 0);
   2157 	txn->mt_dirty_room--;
   2158 }
   2159 
   2160 /** Allocate page numbers and memory for writing.  Maintain me_pglast,
   2161  * me_pghead and mt_next_pgno.  Set #MDB_TXN_ERROR on failure.
   2162  *
   2163  * If there are free pages available from older transactions, they
   2164  * are re-used first. Otherwise allocate a new page at mt_next_pgno.
   2165  * Do not modify the freedB, just merge freeDB records into me_pghead[]
   2166  * and move me_pglast to say which records were consumed.  Only this
   2167  * function can create me_pghead and move me_pglast/mt_next_pgno.
   2168  * @param[in] mc cursor A cursor handle identifying the transaction and
   2169  *	database for which we are allocating.
   2170  * @param[in] num the number of pages to allocate.
   2171  * @param[out] mp Address of the allocated page(s). Requests for multiple pages
   2172  *  will always be satisfied by a single contiguous chunk of memory.
   2173  * @return 0 on success, non-zero on failure.
   2174  */
   2175 static int
   2176 mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
   2177 {
   2178 #ifdef MDB_PARANOID	/* Seems like we can ignore this now */
   2179 	/* Get at most <Max_retries> more freeDB records once me_pghead
   2180 	 * has enough pages.  If not enough, use new pages from the map.
   2181 	 * If <Paranoid> and mc is updating the freeDB, only get new
   2182 	 * records if me_pghead is empty. Then the freelist cannot play
   2183 	 * catch-up with itself by growing while trying to save it.
   2184 	 */
   2185 	enum { Paranoid = 1, Max_retries = 500 };
   2186 #else
   2187 	enum { Paranoid = 0, Max_retries = INT_MAX /*infinite*/ };
   2188 #endif
   2189 	int rc, retry = num * 60;
   2190 	MDB_txn *txn = mc->mc_txn;
   2191 	MDB_env *env = txn->mt_env;
   2192 	pgno_t pgno, *mop = env->me_pghead;
   2193 	unsigned i, j, mop_len = mop ? mop[0] : 0, n2 = num-1;
   2194 	MDB_page *np;
   2195 	txnid_t oldest = 0, last;
   2196 	MDB_cursor_op op;
   2197 	MDB_cursor m2;
   2198 	int found_old = 0;
   2199 
   2200 	/* If there are any loose pages, just use them */
   2201 	if (num == 1 && txn->mt_loose_pgs) {
   2202 		np = txn->mt_loose_pgs;
   2203 		txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np);
   2204 		txn->mt_loose_count--;
   2205 		DPRINTF(("db %d use loose page %"Z"u", DDBI(mc),
   2206 				np->mp_pgno));
   2207 		*mp = np;
   2208 		return MDB_SUCCESS;
   2209 	}
   2210 
   2211 	*mp = NULL;
   2212 
   2213 	/* If our dirty list is already full, we can't do anything */
   2214 	if (txn->mt_dirty_room == 0) {
   2215 		rc = MDB_TXN_FULL;
   2216 		goto fail;
   2217 	}
   2218 
   2219 	for (op = MDB_FIRST;; op = MDB_NEXT) {
   2220 		MDB_val key, data;
   2221 		MDB_node *leaf;
   2222 		pgno_t *idl;
   2223 
   2224 		/* Seek a big enough contiguous page range. Prefer
   2225 		 * pages at the tail, just truncating the list.
   2226 		 */
   2227 		if (mop_len > n2) {
   2228 			i = mop_len;
   2229 			do {
   2230 				pgno = mop[i];
   2231 				if (mop[i-n2] == pgno+n2)
   2232 					goto search_done;
   2233 			} while (--i > n2);
   2234 			if (--retry < 0)
   2235 				break;
   2236 		}
   2237 
   2238 		if (op == MDB_FIRST) {	/* 1st iteration */
   2239 			/* Prepare to fetch more and coalesce */
   2240 			last = env->me_pglast;
   2241 			oldest = env->me_pgoldest;
   2242 			mdb_cursor_init(&m2, txn, FREE_DBI, NULL);
   2243 			if (last) {
   2244 				op = MDB_SET_RANGE;
   2245 				key.mv_data = &last; /* will look up last+1 */
   2246 				key.mv_size = sizeof(last);
   2247 			}
   2248 			if (Paranoid && mc->mc_dbi == FREE_DBI)
   2249 				retry = -1;
   2250 		}
   2251 		if (Paranoid && retry < 0 && mop_len)
   2252 			break;
   2253 
   2254 		last++;
   2255 		/* Do not fetch more if the record will be too recent */
   2256 		if (oldest <= last) {
   2257 			if (!found_old) {
   2258 				oldest = mdb_find_oldest(txn);
   2259 				env->me_pgoldest = oldest;
   2260 				found_old = 1;
   2261 			}
   2262 			if (oldest <= last)
   2263 				break;
   2264 		}
   2265 		rc = mdb_cursor_get(&m2, &key, NULL, op);
   2266 		if (rc) {
   2267 			if (rc == MDB_NOTFOUND)
   2268 				break;
   2269 			goto fail;
   2270 		}
   2271 		last = *(txnid_t*)key.mv_data;
   2272 		if (oldest <= last) {
   2273 			if (!found_old) {
   2274 				oldest = mdb_find_oldest(txn);
   2275 				env->me_pgoldest = oldest;
   2276 				found_old = 1;
   2277 			}
   2278 			if (oldest <= last)
   2279 				break;
   2280 		}
   2281 		np = m2.mc_pg[m2.mc_top];
   2282 		leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]);
   2283 		if ((rc = mdb_node_read(&m2, leaf, &data)) != MDB_SUCCESS)
   2284 			goto fail;
   2285 
   2286 		idl = (MDB_ID *) data.mv_data;
   2287 		i = idl[0];
   2288 		if (!mop) {
   2289 			if (!(env->me_pghead = mop = mdb_midl_alloc(i))) {
   2290 				rc = ENOMEM;
   2291 				goto fail;
   2292 			}
   2293 		} else {
   2294 			if ((rc = mdb_midl_need(&env->me_pghead, i)) != 0)
   2295 				goto fail;
   2296 			mop = env->me_pghead;
   2297 		}
   2298 		env->me_pglast = last;
   2299 #if (MDB_DEBUG) > 1
   2300 		DPRINTF(("IDL read txn %"Z"u root %"Z"u num %u",
   2301 			last, txn->mt_dbs[FREE_DBI].md_root, i));
   2302 		for (j = i; j; j--)
   2303 			DPRINTF(("IDL %"Z"u", idl[j]));
   2304 #endif
   2305 		/* Merge in descending sorted order */
   2306 		mdb_midl_xmerge(mop, idl);
   2307 		mop_len = mop[0];
   2308 	}
   2309 
   2310 	/* Use new pages from the map when nothing suitable in the freeDB */
   2311 	i = 0;
   2312 	pgno = txn->mt_next_pgno;
   2313 	if (pgno + num >= env->me_maxpg) {
   2314 			DPUTS("DB size maxed out");
   2315 			rc = MDB_MAP_FULL;
   2316 			goto fail;
   2317 	}
   2318 
   2319 search_done:
   2320 	if (env->me_flags & MDB_WRITEMAP) {
   2321 		np = (MDB_page *)(env->me_map + env->me_psize * pgno);
   2322 	} else {
   2323 		if (!(np = mdb_page_malloc(txn, num))) {
   2324 			rc = ENOMEM;
   2325 			goto fail;
   2326 		}
   2327 	}
   2328 	if (i) {
   2329 		mop[0] = mop_len -= num;
   2330 		/* Move any stragglers down */
   2331 		for (j = i-num; j < mop_len; )
   2332 			mop[++j] = mop[++i];
   2333 	} else {
   2334 		txn->mt_next_pgno = pgno + num;
   2335 	}
   2336 	np->mp_pgno = pgno;
   2337 	mdb_page_dirty(txn, np);
   2338 	*mp = np;
   2339 
   2340 	return MDB_SUCCESS;
   2341 
   2342 fail:
   2343 	txn->mt_flags |= MDB_TXN_ERROR;
   2344 	return rc;
   2345 }
   2346 
   2347 /** Copy the used portions of a non-overflow page.
   2348  * @param[in] dst page to copy into
   2349  * @param[in] src page to copy from
   2350  * @param[in] psize size of a page
   2351  */
   2352 static void
   2353 mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize)
   2354 {
   2355 	enum { Align = sizeof(pgno_t) };
   2356 	indx_t upper = src->mp_upper, lower = src->mp_lower, unused = upper-lower;
   2357 
   2358 	/* If page isn't full, just copy the used portion. Adjust
   2359 	 * alignment so memcpy may copy words instead of bytes.
   2360 	 */
   2361 	if ((unused &= -Align) && !IS_LEAF2(src)) {
   2362 		upper = (upper + PAGEBASE) & -Align;
   2363 		memcpy(dst, src, (lower + PAGEBASE + (Align-1)) & -Align);
   2364 		memcpy((pgno_t *)((char *)dst+upper), (pgno_t *)((char *)src+upper),
   2365 			psize - upper);
   2366 	} else {
   2367 		memcpy(dst, src, psize - unused);
   2368 	}
   2369 }
   2370 
   2371 /** Pull a page off the txn's spill list, if present.
   2372  * If a page being referenced was spilled to disk in this txn, bring
   2373  * it back and make it dirty/writable again.
   2374  * @param[in] txn the transaction handle.
   2375  * @param[in] mp the page being referenced. It must not be dirty.
   2376  * @param[out] ret the writable page, if any. ret is unchanged if
   2377  * mp wasn't spilled.
   2378  */
   2379 static int
   2380 mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
   2381 {
   2382 	MDB_env *env = txn->mt_env;
   2383 	const MDB_txn *tx2;
   2384 	unsigned x;
   2385 	pgno_t pgno = mp->mp_pgno, pn = pgno << 1;
   2386 
   2387 	for (tx2 = txn; tx2; tx2=tx2->mt_parent) {
   2388 		if (!tx2->mt_spill_pgs)
   2389 			continue;
   2390 		x = mdb_midl_search(tx2->mt_spill_pgs, pn);
   2391 		if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) {
   2392 			MDB_page *np;
   2393 			int num;
   2394 			if (txn->mt_dirty_room == 0)
   2395 				return MDB_TXN_FULL;
   2396 			if (IS_OVERFLOW(mp))
   2397 				num = mp->mp_pages;
   2398 			else
   2399 				num = 1;
   2400 			if (env->me_flags & MDB_WRITEMAP) {
   2401 				np = mp;
   2402 			} else {
   2403 				np = mdb_page_malloc(txn, num);
   2404 				if (!np)
   2405 					return ENOMEM;
   2406 				if (num > 1)
   2407 					memcpy(np, mp, num * env->me_psize);
   2408 				else
   2409 					mdb_page_copy(np, mp, env->me_psize);
   2410 			}
   2411 			if (tx2 == txn) {
   2412 				/* If in current txn, this page is no longer spilled.
   2413 				 * If it happens to be the last page, truncate the spill list.
   2414 				 * Otherwise mark it as deleted by setting the LSB.
   2415 				 */
   2416 				if (x == txn->mt_spill_pgs[0])
   2417 					txn->mt_spill_pgs[0]--;
   2418 				else
   2419 					txn->mt_spill_pgs[x] |= 1;
   2420 			}	/* otherwise, if belonging to a parent txn, the
   2421 				 * page remains spilled until child commits
   2422 				 */
   2423 
   2424 			mdb_page_dirty(txn, np);
   2425 			np->mp_flags |= P_DIRTY;
   2426 			*ret = np;
   2427 			break;
   2428 		}
   2429 	}
   2430 	return MDB_SUCCESS;
   2431 }
   2432 
   2433 /** Touch a page: make it dirty and re-insert into tree with updated pgno.
   2434  * Set #MDB_TXN_ERROR on failure.
   2435  * @param[in] mc cursor pointing to the page to be touched
   2436  * @return 0 on success, non-zero on failure.
   2437  */
   2438 static int
   2439 mdb_page_touch(MDB_cursor *mc)
   2440 {
   2441 	MDB_page *mp = mc->mc_pg[mc->mc_top], *np;
   2442 	MDB_txn *txn = mc->mc_txn;
   2443 	MDB_cursor *m2, *m3;
   2444 	pgno_t	pgno;
   2445 	int rc;
   2446 
   2447 	if (!F_ISSET(MP_FLAGS(mp), P_DIRTY)) {
   2448 		if (txn->mt_flags & MDB_TXN_SPILLS) {
   2449 			np = NULL;
   2450 			rc = mdb_page_unspill(txn, mp, &np);
   2451 			if (rc)
   2452 				goto fail;
   2453 			if (np)
   2454 				goto done;
   2455 		}
   2456 		if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) ||
   2457 			(rc = mdb_page_alloc(mc, 1, &np)))
   2458 			goto fail;
   2459 		pgno = np->mp_pgno;
   2460 		DPRINTF(("touched db %d page %"Z"u -> %"Z"u", DDBI(mc),
   2461 			mp->mp_pgno, pgno));
   2462 		mdb_cassert(mc, mp->mp_pgno != pgno);
   2463 		mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno);
   2464 		/* Update the parent page, if any, to point to the new page */
   2465 		if (mc->mc_top) {
   2466 			MDB_page *parent = mc->mc_pg[mc->mc_top-1];
   2467 			MDB_node *node = NODEPTR(parent, mc->mc_ki[mc->mc_top-1]);
   2468 			SETPGNO(node, pgno);
   2469 		} else {
   2470 			mc->mc_db->md_root = pgno;
   2471 		}
   2472 	} else if (txn->mt_parent && !IS_SUBP(mp)) {
   2473 		MDB_ID2 mid, *dl = txn->mt_u.dirty_list;
   2474 		pgno = mp->mp_pgno;
   2475 		/* If txn has a parent, make sure the page is in our
   2476 		 * dirty list.
   2477 		 */
   2478 		if (dl[0].mid) {
   2479 			unsigned x = mdb_mid2l_search(dl, pgno);
   2480 			if (x <= dl[0].mid && dl[x].mid == pgno) {
   2481 				if (mp != dl[x].mptr) { /* bad cursor? */
   2482 					mc->mc_flags &= ~(C_INITIALIZED|C_EOF);
   2483 					txn->mt_flags |= MDB_TXN_ERROR;
   2484 					return MDB_CORRUPTED;
   2485 				}
   2486 				return 0;
   2487 			}
   2488 		}
   2489 		mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX);
   2490 		/* No - copy it */
   2491 		np = mdb_page_malloc(txn, 1);
   2492 		if (!np)
   2493 			return ENOMEM;
   2494 		mid.mid = pgno;
   2495 		mid.mptr = np;
   2496 		rc = mdb_mid2l_insert(dl, &mid);
   2497 		mdb_cassert(mc, rc == 0);
   2498 	} else {
   2499 		return 0;
   2500 	}
   2501 
   2502 	mdb_page_copy(np, mp, txn->mt_env->me_psize);
   2503 	np->mp_pgno = pgno;
   2504 	np->mp_flags |= P_DIRTY;
   2505 
   2506 done:
   2507 	/* Adjust cursors pointing to mp */
   2508 	mc->mc_pg[mc->mc_top] = np;
   2509 	m2 = txn->mt_cursors[mc->mc_dbi];
   2510 	if (mc->mc_flags & C_SUB) {
   2511 		for (; m2; m2=m2->mc_next) {
   2512 			m3 = &m2->mc_xcursor->mx_cursor;
   2513 			if (m3->mc_snum < mc->mc_snum) continue;
   2514 			if (m3->mc_pg[mc->mc_top] == mp)
   2515 				m3->mc_pg[mc->mc_top] = np;
   2516 		}
   2517 	} else {
   2518 		for (; m2; m2=m2->mc_next) {
   2519 			if (m2->mc_snum < mc->mc_snum) continue;
   2520 			if (m2 == mc) continue;
   2521 			if (m2->mc_pg[mc->mc_top] == mp) {
   2522 				m2->mc_pg[mc->mc_top] = np;
   2523 				if (IS_LEAF(np))
   2524 					XCURSOR_REFRESH(m2, mc->mc_top, np);
   2525 			}
   2526 		}
   2527 	}
   2528 	return 0;
   2529 
   2530 fail:
   2531 	txn->mt_flags |= MDB_TXN_ERROR;
   2532 	return rc;
   2533 }
   2534 
   2535 int
   2536 mdb_env_sync(MDB_env *env, int force)
   2537 {
   2538 	int rc = 0;
   2539 	if (env->me_flags & MDB_RDONLY)
   2540 		return EACCES;
   2541 	if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) {
   2542 		if (env->me_flags & MDB_WRITEMAP) {
   2543 			int flags = ((env->me_flags & MDB_MAPASYNC) && !force)
   2544 				? MS_ASYNC : MS_SYNC;
   2545 			if (MDB_MSYNC(env->me_map, env->me_mapsize, flags))
   2546 				rc = ErrCode();
   2547 #ifdef _WIN32
   2548 			else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd))
   2549 				rc = ErrCode();
   2550 #endif
   2551 		} else {
   2552 #ifdef BROKEN_FDATASYNC
   2553 			if (env->me_flags & MDB_FSYNCONLY) {
   2554 				if (fsync(env->me_fd))
   2555 					rc = ErrCode();
   2556 			} else
   2557 #endif
   2558 			if (MDB_FDATASYNC(env->me_fd))
   2559 				rc = ErrCode();
   2560 		}
   2561 	}
   2562 	return rc;
   2563 }
   2564 
   2565 /** Back up parent txn's cursors, then grab the originals for tracking */
   2566 static int
   2567 mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst)
   2568 {
   2569 	MDB_cursor *mc, *bk;
   2570 	MDB_xcursor *mx;
   2571 	size_t size;
   2572 	int i;
   2573 
   2574 	for (i = src->mt_numdbs; --i >= 0; ) {
   2575 		if ((mc = src->mt_cursors[i]) != NULL) {
   2576 			size = sizeof(MDB_cursor);
   2577 			if (mc->mc_xcursor)
   2578 				size += sizeof(MDB_xcursor);
   2579 			for (; mc; mc = bk->mc_next) {
   2580 				bk = malloc(size);
   2581 				if (!bk)
   2582 					return ENOMEM;
   2583 				*bk = *mc;
   2584 				mc->mc_backup = bk;
   2585 				mc->mc_db = &dst->mt_dbs[i];
   2586 				/* Kill pointers into src to reduce abuse: The
   2587 				 * user may not use mc until dst ends. But we need a valid
   2588 				 * txn pointer here for cursor fixups to keep working.
   2589 				 */
   2590 				mc->mc_txn    = dst;
   2591 				mc->mc_dbflag = &dst->mt_dbflags[i];
   2592 				if ((mx = mc->mc_xcursor) != NULL) {
   2593 					*(MDB_xcursor *)(bk+1) = *mx;
   2594 					mx->mx_cursor.mc_txn = dst;
   2595 				}
   2596 				mc->mc_next = dst->mt_cursors[i];
   2597 				dst->mt_cursors[i] = mc;
   2598 			}
   2599 		}
   2600 	}
   2601 	return MDB_SUCCESS;
   2602 }
   2603 
   2604 /** Close this write txn's cursors, give parent txn's cursors back to parent.
   2605  * @param[in] txn the transaction handle.
   2606  * @param[in] merge true to keep changes to parent cursors, false to revert.
   2607  * @return 0 on success, non-zero on failure.
   2608  */
   2609 static void
   2610 mdb_cursors_close(MDB_txn *txn, unsigned merge)
   2611 {
   2612 	MDB_cursor **cursors = txn->mt_cursors, *mc, *next, *bk;
   2613 	MDB_xcursor *mx;
   2614 	int i;
   2615 
   2616 	for (i = txn->mt_numdbs; --i >= 0; ) {
   2617 		for (mc = cursors[i]; mc; mc = next) {
   2618 			next = mc->mc_next;
   2619 			if ((bk = mc->mc_backup) != NULL) {
   2620 				if (merge) {
   2621 					/* Commit changes to parent txn */
   2622 					mc->mc_next = bk->mc_next;
   2623 					mc->mc_backup = bk->mc_backup;
   2624 					mc->mc_txn = bk->mc_txn;
   2625 					mc->mc_db = bk->mc_db;
   2626 					mc->mc_dbflag = bk->mc_dbflag;
   2627 					if ((mx = mc->mc_xcursor) != NULL)
   2628 						mx->mx_cursor.mc_txn = bk->mc_txn;
   2629 				} else {
   2630 					/* Abort nested txn */
   2631 					*mc = *bk;
   2632 					if ((mx = mc->mc_xcursor) != NULL)
   2633 						*mx = *(MDB_xcursor *)(bk+1);
   2634 				}
   2635 				mc = bk;
   2636 			}
   2637 			/* Only malloced cursors are permanently tracked. */
   2638 			free(mc);
   2639 		}
   2640 		cursors[i] = NULL;
   2641 	}
   2642 }
   2643 
   2644 #if !(MDB_PIDLOCK)		/* Currently the same as defined(_WIN32) */
   2645 enum Pidlock_op {
   2646 	Pidset, Pidcheck
   2647 };
   2648 #else
   2649 enum Pidlock_op {
   2650 	Pidset = F_SETLK, Pidcheck = F_GETLK
   2651 };
   2652 #endif
   2653 
   2654 /** Set or check a pid lock. Set returns 0 on success.
   2655  * Check returns 0 if the process is certainly dead, nonzero if it may
   2656  * be alive (the lock exists or an error happened so we do not know).
   2657  *
   2658  * On Windows Pidset is a no-op, we merely check for the existence
   2659  * of the process with the given pid. On POSIX we use a single byte
   2660  * lock on the lockfile, set at an offset equal to the pid.
   2661  */
   2662 static int
   2663 mdb_reader_pid(MDB_env *env, enum Pidlock_op op, MDB_PID_T pid)
   2664 {
   2665 #if !(MDB_PIDLOCK)		/* Currently the same as defined(_WIN32) */
   2666 	int ret = 0;
   2667 	HANDLE h;
   2668 	if (op == Pidcheck) {
   2669 		h = OpenProcess(env->me_pidquery, FALSE, pid);
   2670 		/* No documented "no such process" code, but other program use this: */
   2671 		if (!h)
   2672 			return ErrCode() != ERROR_INVALID_PARAMETER;
   2673 		/* A process exists until all handles to it close. Has it exited? */
   2674 		ret = WaitForSingleObject(h, 0) != 0;
   2675 		CloseHandle(h);
   2676 	}
   2677 	return ret;
   2678 #else
   2679 	for (;;) {
   2680 		int rc;
   2681 		struct flock lock_info;
   2682 		memset(&lock_info, 0, sizeof(lock_info));
   2683 		lock_info.l_type = F_WRLCK;
   2684 		lock_info.l_whence = SEEK_SET;
   2685 		lock_info.l_start = pid;
   2686 		lock_info.l_len = 1;
   2687 		if ((rc = fcntl(env->me_lfd, op, &lock_info)) == 0) {
   2688 			if (op == F_GETLK && lock_info.l_type != F_UNLCK)
   2689 				rc = -1;
   2690 		} else if ((rc = ErrCode()) == EINTR) {
   2691 			continue;
   2692 		}
   2693 		return rc;
   2694 	}
   2695 #endif
   2696 }
   2697 
   2698 /** Common code for #mdb_txn_begin() and #mdb_txn_renew().
   2699  * @param[in] txn the transaction handle to initialize
   2700  * @return 0 on success, non-zero on failure.
   2701  */
   2702 static int
   2703 mdb_txn_renew0(MDB_txn *txn)
   2704 {
   2705 	MDB_env *env = txn->mt_env;
   2706 	MDB_txninfo *ti = env->me_txns;
   2707 	MDB_meta *meta;
   2708 	unsigned int i, nr, flags = txn->mt_flags;
   2709 	uint16_t x;
   2710 	int rc, new_notls = 0;
   2711 
   2712 	if ((flags &= MDB_TXN_RDONLY) != 0) {
   2713 		if (!ti) {
   2714 			meta = mdb_env_pick_meta(env);
   2715 			txn->mt_txnid = meta->mm_txnid;
   2716 			txn->mt_u.reader = NULL;
   2717 		} else {
   2718 			MDB_reader *r = (env->me_flags & MDB_NOTLS) ? txn->mt_u.reader :
   2719 				pthread_getspecific(env->me_txkey);
   2720 			if (r) {
   2721 				if (r->mr_pid != env->me_pid || r->mr_txnid != (txnid_t)-1)
   2722 					return MDB_BAD_RSLOT;
   2723 			} else {
   2724 				MDB_PID_T pid = env->me_pid;
   2725 				MDB_THR_T tid = pthread_self();
   2726 				mdb_mutexref_t rmutex = env->me_rmutex;
   2727 
   2728 				if (!env->me_live_reader) {
   2729 					rc = mdb_reader_pid(env, Pidset, pid);
   2730 					if (rc)
   2731 						return rc;
   2732 					env->me_live_reader = 1;
   2733 				}
   2734 
   2735 				if (LOCK_MUTEX(rc, env, rmutex))
   2736 					return rc;
   2737 				nr = ti->mti_numreaders;
   2738 				for (i=0; i<nr; i++)
   2739 					if (ti->mti_readers[i].mr_pid == 0)
   2740 						break;
   2741 				if (i == env->me_maxreaders) {
   2742 					UNLOCK_MUTEX(rmutex);
   2743 					return MDB_READERS_FULL;
   2744 				}
   2745 				r = &ti->mti_readers[i];
   2746 				/* Claim the reader slot, carefully since other code
   2747 				 * uses the reader table un-mutexed: First reset the
   2748 				 * slot, next publish it in mti_numreaders.  After
   2749 				 * that, it is safe for mdb_env_close() to touch it.
   2750 				 * When it will be closed, we can finally claim it.
   2751 				 */
   2752 				r->mr_pid = 0;
   2753 				r->mr_txnid = (txnid_t)-1;
   2754 				r->mr_tid = tid;
   2755 				if (i == nr)
   2756 					ti->mti_numreaders = ++nr;
   2757 				env->me_close_readers = nr;
   2758 				r->mr_pid = pid;
   2759 				UNLOCK_MUTEX(rmutex);
   2760 
   2761 				new_notls = (env->me_flags & MDB_NOTLS);
   2762 				if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) {
   2763 					r->mr_pid = 0;
   2764 					return rc;
   2765 				}
   2766 			}
   2767 			do /* LY: Retry on a race, ITS#7970. */
   2768 				r->mr_txnid = ti->mti_txnid;
   2769 			while(r->mr_txnid != ti->mti_txnid);
   2770 			txn->mt_txnid = r->mr_txnid;
   2771 			txn->mt_u.reader = r;
   2772 			meta = env->me_metas[txn->mt_txnid & 1];
   2773 		}
   2774 
   2775 	} else {
   2776 		/* Not yet touching txn == env->me_txn0, it may be active */
   2777 		if (ti) {
   2778 			if (LOCK_MUTEX(rc, env, env->me_wmutex))
   2779 				return rc;
   2780 			txn->mt_txnid = ti->mti_txnid;
   2781 			meta = env->me_metas[txn->mt_txnid & 1];
   2782 		} else {
   2783 			meta = mdb_env_pick_meta(env);
   2784 			txn->mt_txnid = meta->mm_txnid;
   2785 		}
   2786 		txn->mt_txnid++;
   2787 #if MDB_DEBUG
   2788 		if (txn->mt_txnid == mdb_debug_start)
   2789 			mdb_debug = 1;
   2790 #endif
   2791 		txn->mt_child = NULL;
   2792 		txn->mt_loose_pgs = NULL;
   2793 		txn->mt_loose_count = 0;
   2794 		txn->mt_dirty_room = MDB_IDL_UM_MAX;
   2795 		txn->mt_u.dirty_list = env->me_dirty_list;
   2796 		txn->mt_u.dirty_list[0].mid = 0;
   2797 		txn->mt_free_pgs = env->me_free_pgs;
   2798 		txn->mt_free_pgs[0] = 0;
   2799 		txn->mt_spill_pgs = NULL;
   2800 		env->me_txn = txn;
   2801 		memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int));
   2802 	}
   2803 
   2804 	/* Copy the DB info and flags */
   2805 	memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDB_db));
   2806 
   2807 	/* Moved to here to avoid a data race in read TXNs */
   2808 	txn->mt_next_pgno = meta->mm_last_pg+1;
   2809 
   2810 	txn->mt_flags = flags;
   2811 
   2812 	/* Setup db info */
   2813 	txn->mt_numdbs = env->me_numdbs;
   2814 	for (i=CORE_DBS; i<txn->mt_numdbs; i++) {
   2815 		x = env->me_dbflags[i];
   2816 		txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS;
   2817 		txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_USRVALID|DB_STALE : 0;
   2818 	}
   2819 	txn->mt_dbflags[MAIN_DBI] = DB_VALID|DB_USRVALID;
   2820 	txn->mt_dbflags[FREE_DBI] = DB_VALID;
   2821 
   2822 	if (env->me_flags & MDB_FATAL_ERROR) {
   2823 		DPUTS("environment had fatal error, must shutdown!");
   2824 		rc = MDB_PANIC;
   2825 	} else if (env->me_maxpg < txn->mt_next_pgno) {
   2826 		rc = MDB_MAP_RESIZED;
   2827 	} else {
   2828 		return MDB_SUCCESS;
   2829 	}
   2830 	mdb_txn_end(txn, new_notls /*0 or MDB_END_SLOT*/ | MDB_END_FAIL_BEGIN);
   2831 	return rc;
   2832 }
   2833 
   2834 int
   2835 mdb_txn_renew(MDB_txn *txn)
   2836 {
   2837 	int rc;
   2838 
   2839 	if (!txn || !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY|MDB_TXN_FINISHED))
   2840 		return EINVAL;
   2841 
   2842 	rc = mdb_txn_renew0(txn);
   2843 	if (rc == MDB_SUCCESS) {
   2844 		DPRINTF(("renew txn %"Z"u%c %p on mdbenv %p, root page %"Z"u",
   2845 			txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w',
   2846 			(void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root));
   2847 	}
   2848 	return rc;
   2849 }
   2850 
   2851 int
   2852 mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
   2853 {
   2854 	MDB_txn *txn;
   2855 	MDB_ntxn *ntxn;
   2856 	int rc, size, tsize;
   2857 
   2858 	flags &= MDB_TXN_BEGIN_FLAGS;
   2859 	flags |= env->me_flags & MDB_WRITEMAP;
   2860 
   2861 	if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */
   2862 		return EACCES;
   2863 
   2864 	if (parent) {
   2865 		/* Nested transactions: Max 1 child, write txns only, no writemap */
   2866 		flags |= parent->mt_flags;
   2867 		if (flags & (MDB_RDONLY|MDB_WRITEMAP|MDB_TXN_BLOCKED)) {
   2868 			return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN;
   2869 		}
   2870 		/* Child txns save MDB_pgstate and use own copy of cursors */
   2871 		size = env->me_maxdbs * (sizeof(MDB_db)+sizeof(MDB_cursor *)+1);
   2872 		size += tsize = sizeof(MDB_ntxn);
   2873 	} else if (flags & MDB_RDONLY) {
   2874 		size = env->me_maxdbs * (sizeof(MDB_db)+1);
   2875 		size += tsize = sizeof(MDB_txn);
   2876 	} else {
   2877 		/* Reuse preallocated write txn. However, do not touch it until
   2878 		 * mdb_txn_renew0() succeeds, since it currently may be active.
   2879 		 */
   2880 		txn = env->me_txn0;
   2881 		goto renew;
   2882 	}
   2883 	if ((txn = calloc(1, size)) == NULL) {
   2884 		DPRINTF(("calloc: %s", strerror(errno)));
   2885 		return ENOMEM;
   2886 	}
   2887 	txn->mt_dbxs = env->me_dbxs;	/* static */
   2888 	txn->mt_dbs = (MDB_db *) ((char *)txn + tsize);
   2889 	txn->mt_dbflags = (unsigned char *)txn + size - env->me_maxdbs;
   2890 	txn->mt_flags = flags;
   2891 	txn->mt_env = env;
   2892 
   2893 	if (parent) {
   2894 		unsigned int i;
   2895 		txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs);
   2896 		txn->mt_dbiseqs = parent->mt_dbiseqs;
   2897 		txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE);
   2898 		if (!txn->mt_u.dirty_list ||
   2899 			!(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)))
   2900 		{
   2901 			free(txn->mt_u.dirty_list);
   2902 			free(txn);
   2903 			return ENOMEM;
   2904 		}
   2905 		txn->mt_txnid = parent->mt_txnid;
   2906 		txn->mt_dirty_room = parent->mt_dirty_room;
   2907 		txn->mt_u.dirty_list[0].mid = 0;
   2908 		txn->mt_spill_pgs = NULL;
   2909 		txn->mt_next_pgno = parent->mt_next_pgno;
   2910 		parent->mt_flags |= MDB_TXN_HAS_CHILD;
   2911 		parent->mt_child = txn;
   2912 		txn->mt_parent = parent;
   2913 		txn->mt_numdbs = parent->mt_numdbs;
   2914 		memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db));
   2915 		/* Copy parent's mt_dbflags, but clear DB_NEW */
   2916 		for (i=0; i<txn->mt_numdbs; i++)
   2917 			txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~DB_NEW;
   2918 		rc = 0;
   2919 		ntxn = (MDB_ntxn *)txn;
   2920 		ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */
   2921 		if (env->me_pghead) {
   2922 			size = MDB_IDL_SIZEOF(env->me_pghead);
   2923 			env->me_pghead = mdb_midl_alloc(env->me_pghead[0]);
   2924 			if (env->me_pghead)
   2925 				memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size);
   2926 			else
   2927 				rc = ENOMEM;
   2928 		}
   2929 		if (!rc)
   2930 			rc = mdb_cursor_shadow(parent, txn);
   2931 		if (rc)
   2932 			mdb_txn_end(txn, MDB_END_FAIL_BEGINCHILD);
   2933 	} else { /* MDB_RDONLY */
   2934 		txn->mt_dbiseqs = env->me_dbiseqs;
   2935 renew:
   2936 		rc = mdb_txn_renew0(txn);
   2937 	}
   2938 	if (rc) {
   2939 		if (txn != env->me_txn0)
   2940 			free(txn);
   2941 	} else {
   2942 		txn->mt_flags |= flags;	/* could not change txn=me_txn0 earlier */
   2943 		*ret = txn;
   2944 		DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u",
   2945 			txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w',
   2946 			(void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root));
   2947 	}
   2948 
   2949 	return rc;
   2950 }
   2951 
   2952 MDB_env *
   2953 mdb_txn_env(MDB_txn *txn)
   2954 {
   2955 	if(!txn) return NULL;
   2956 	return txn->mt_env;
   2957 }
   2958 
   2959 size_t
   2960 mdb_txn_id(MDB_txn *txn)
   2961 {
   2962     if(!txn) return 0;
   2963     return txn->mt_txnid;
   2964 }
   2965 
   2966 /** Export or close DBI handles opened in this txn. */
   2967 static void
   2968 mdb_dbis_update(MDB_txn *txn, int keep)
   2969 {
   2970 	int i;
   2971 	MDB_dbi n = txn->mt_numdbs;
   2972 	MDB_env *env = txn->mt_env;
   2973 	unsigned char *tdbflags = txn->mt_dbflags;
   2974 
   2975 	for (i = n; --i >= CORE_DBS;) {
   2976 		if (tdbflags[i] & DB_NEW) {
   2977 			if (keep) {
   2978 				env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID;
   2979 			} else {
   2980 				char *ptr = env->me_dbxs[i].md_name.mv_data;
   2981 				if (ptr) {
   2982 					env->me_dbxs[i].md_name.mv_data = NULL;
   2983 					env->me_dbxs[i].md_name.mv_size = 0;
   2984 					env->me_dbflags[i] = 0;
   2985 					env->me_dbiseqs[i]++;
   2986 					free(ptr);
   2987 				}
   2988 			}
   2989 		}
   2990 	}
   2991 	if (keep && env->me_numdbs < n)
   2992 		env->me_numdbs = n;
   2993 }
   2994 
   2995 /** End a transaction, except successful commit of a nested transaction.
   2996  * May be called twice for readonly txns: First reset it, then abort.
   2997  * @param[in] txn the transaction handle to end
   2998  * @param[in] mode why and how to end the transaction
   2999  */
   3000 static void
   3001 mdb_txn_end(MDB_txn *txn, unsigned mode)
   3002 {
   3003 	MDB_env	*env = txn->mt_env;
   3004 #if MDB_DEBUG
   3005 	static const char *const names[] = MDB_END_NAMES;
   3006 #endif
   3007 
   3008 	/* Export or close DBI handles opened in this txn */
   3009 	mdb_dbis_update(txn, mode & MDB_END_UPDATE);
   3010 
   3011 	DPRINTF(("%s txn %"Z"u%c %p on mdbenv %p, root page %"Z"u",
   3012 		names[mode & MDB_END_OPMASK],
   3013 		txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w',
   3014 		(void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root));
   3015 
   3016 	if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) {
   3017 		if (txn->mt_u.reader) {
   3018 			txn->mt_u.reader->mr_txnid = (txnid_t)-1;
   3019 			if (!(env->me_flags & MDB_NOTLS)) {
   3020 				txn->mt_u.reader = NULL; /* txn does not own reader */
   3021 			} else if (mode & MDB_END_SLOT) {
   3022 				txn->mt_u.reader->mr_pid = 0;
   3023 				txn->mt_u.reader = NULL;
   3024 			} /* else txn owns the slot until it does MDB_END_SLOT */
   3025 		}
   3026 		txn->mt_numdbs = 0;		/* prevent further DBI activity */
   3027 		txn->mt_flags |= MDB_TXN_FINISHED;
   3028 
   3029 	} else if (!F_ISSET(txn->mt_flags, MDB_TXN_FINISHED)) {
   3030 		pgno_t *pghead = env->me_pghead;
   3031 
   3032 		if (!(mode & MDB_END_UPDATE)) /* !(already closed cursors) */
   3033 			mdb_cursors_close(txn, 0);
   3034 		if (!(env->me_flags & MDB_WRITEMAP)) {
   3035 			mdb_dlist_free(txn);
   3036 		}
   3037 
   3038 		txn->mt_numdbs = 0;
   3039 		txn->mt_flags = MDB_TXN_FINISHED;
   3040 
   3041 		if (!txn->mt_parent) {
   3042 			mdb_midl_shrink(&txn->mt_free_pgs);
   3043 			env->me_free_pgs = txn->mt_free_pgs;
   3044 			/* me_pgstate: */
   3045 			env->me_pghead = NULL;
   3046 			env->me_pglast = 0;
   3047 
   3048 			env->me_txn = NULL;
   3049 			mode = 0;	/* txn == env->me_txn0, do not free() it */
   3050 
   3051 			/* The writer mutex was locked in mdb_txn_begin. */
   3052 			if (env->me_txns)
   3053 				UNLOCK_MUTEX(env->me_wmutex);
   3054 		} else {
   3055 			txn->mt_parent->mt_child = NULL;
   3056 			txn->mt_parent->mt_flags &= ~MDB_TXN_HAS_CHILD;
   3057 			env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate;
   3058 			mdb_midl_free(txn->mt_free_pgs);
   3059 			free(txn->mt_u.dirty_list);
   3060 		}
   3061 		mdb_midl_free(txn->mt_spill_pgs);
   3062 
   3063 		mdb_midl_free(pghead);
   3064 	}
   3065 
   3066 	if (mode & MDB_END_FREE)
   3067 		free(txn);
   3068 }
   3069 
   3070 void
   3071 mdb_txn_reset(MDB_txn *txn)
   3072 {
   3073 	if (txn == NULL)
   3074 		return;
   3075 
   3076 	/* This call is only valid for read-only txns */
   3077 	if (!(txn->mt_flags & MDB_TXN_RDONLY))
   3078 		return;
   3079 
   3080 	mdb_txn_end(txn, MDB_END_RESET);
   3081 }
   3082 
   3083 void
   3084 mdb_txn_abort(MDB_txn *txn)
   3085 {
   3086 	if (txn == NULL)
   3087 		return;
   3088 
   3089 	if (txn->mt_child)
   3090 		mdb_txn_abort(txn->mt_child);
   3091 
   3092 	mdb_txn_end(txn, MDB_END_ABORT|MDB_END_SLOT|MDB_END_FREE);
   3093 }
   3094 
   3095 /** Save the freelist as of this transaction to the freeDB.
   3096  * This changes the freelist. Keep trying until it stabilizes.
   3097  */
   3098 static int
   3099 mdb_freelist_save(MDB_txn *txn)
   3100 {
   3101 	/* env->me_pghead[] can grow and shrink during this call.
   3102 	 * env->me_pglast and txn->mt_free_pgs[] can only grow.
   3103 	 * Page numbers cannot disappear from txn->mt_free_pgs[].
   3104 	 */
   3105 	MDB_cursor mc;
   3106 	MDB_env	*env = txn->mt_env;
   3107 	int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1;
   3108 	txnid_t	pglast = 0, head_id = 0;
   3109 	pgno_t	freecnt = 0, *free_pgs, *mop;
   3110 	ssize_t	head_room = 0, total_room = 0, mop_len, clean_limit;
   3111 
   3112 	mdb_cursor_init(&mc, txn, FREE_DBI, NULL);
   3113 
   3114 	if (env->me_pghead) {
   3115 		/* Make sure first page of freeDB is touched and on freelist */
   3116 		rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST|MDB_PS_MODIFY);
   3117 		if (rc && rc != MDB_NOTFOUND)
   3118 			return rc;
   3119 	}
   3120 
   3121 	if (!env->me_pghead && txn->mt_loose_pgs) {
   3122 		/* Put loose page numbers in mt_free_pgs, since
   3123 		 * we may be unable to return them to me_pghead.
   3124 		 */
   3125 		MDB_page *mp = txn->mt_loose_pgs;
   3126 		MDB_ID2 *dl = txn->mt_u.dirty_list;
   3127 		unsigned x;
   3128 		if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0)
   3129 			return rc;
   3130 		for (; mp; mp = NEXT_LOOSE_PAGE(mp)) {
   3131 			mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno);
   3132 			/* must also remove from dirty list */
   3133 			if (txn->mt_flags & MDB_TXN_WRITEMAP) {
   3134 				for (x=1; x<=dl[0].mid; x++)
   3135 					if (dl[x].mid == mp->mp_pgno)
   3136 						break;
   3137 				mdb_tassert(txn, x <= dl[0].mid);
   3138 			} else {
   3139 				x = mdb_mid2l_search(dl, mp->mp_pgno);
   3140 				mdb_tassert(txn, dl[x].mid == mp->mp_pgno);
   3141 				mdb_dpage_free(env, mp);
   3142 			}
   3143 			dl[x].mptr = NULL;
   3144 		}
   3145 		{
   3146 			/* squash freed slots out of the dirty list */
   3147 			unsigned y;
   3148 			for (y=1; dl[y].mptr && y <= dl[0].mid; y++);
   3149 			if (y <= dl[0].mid) {
   3150 				for(x=y, y++;;) {
   3151 					while (!dl[y].mptr && y <= dl[0].mid) y++;
   3152 					if (y > dl[0].mid) break;
   3153 					dl[x++] = dl[y++];
   3154 				}
   3155 				dl[0].mid = x-1;
   3156 			} else {
   3157 				/* all slots freed */
   3158 				dl[0].mid = 0;
   3159 			}
   3160 		}
   3161 		txn->mt_loose_pgs = NULL;
   3162 		txn->mt_loose_count = 0;
   3163 	}
   3164 
   3165 	/* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */
   3166 	clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP))
   3167 		? SSIZE_MAX : maxfree_1pg;
   3168 
   3169 	for (;;) {
   3170 		/* Come back here after each Put() in case freelist changed */
   3171 		MDB_val key, data;
   3172 		pgno_t *pgs;
   3173 		ssize_t j;
   3174 
   3175 		/* If using records from freeDB which we have not yet
   3176 		 * deleted, delete them and any we reserved for me_pghead.
   3177 		 */
   3178 		while (pglast < env->me_pglast) {
   3179 			rc = mdb_cursor_first(&mc, &key, NULL);
   3180 			if (rc)
   3181 				return rc;
   3182 			pglast = head_id = *(txnid_t *)key.mv_data;
   3183 			total_room = head_room = 0;
   3184 			mdb_tassert(txn, pglast <= env->me_pglast);
   3185 			rc = mdb_cursor_del(&mc, 0);
   3186 			if (rc)
   3187 				return rc;
   3188 		}
   3189 
   3190 		/* Save the IDL of pages freed by this txn, to a single record */
   3191 		if (freecnt < txn->mt_free_pgs[0]) {
   3192 			if (!freecnt) {
   3193 				/* Make sure last page of freeDB is touched and on freelist */
   3194 				rc = mdb_page_search(&mc, NULL, MDB_PS_LAST|MDB_PS_MODIFY);
   3195 				if (rc && rc != MDB_NOTFOUND)
   3196 					return rc;
   3197 			}
   3198 			free_pgs = txn->mt_free_pgs;
   3199 			/* Write to last page of freeDB */
   3200 			key.mv_size = sizeof(txn->mt_txnid);
   3201 			key.mv_data = &txn->mt_txnid;
   3202 			do {
   3203 				freecnt = free_pgs[0];
   3204 				data.mv_size = MDB_IDL_SIZEOF(free_pgs);
   3205 				rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE);
   3206 				if (rc)
   3207 					return rc;
   3208 				/* Retry if mt_free_pgs[] grew during the Put() */
   3209 				free_pgs = txn->mt_free_pgs;
   3210 			} while (freecnt < free_pgs[0]);
   3211 			mdb_midl_sort(free_pgs);
   3212 			memcpy(data.mv_data, free_pgs, data.mv_size);
   3213 #if (MDB_DEBUG) > 1
   3214 			{
   3215 				unsigned int i = free_pgs[0];
   3216 				DPRINTF(("IDL write txn %"Z"u root %"Z"u num %u",
   3217 					txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i));
   3218 				for (; i; i--)
   3219 					DPRINTF(("IDL %"Z"u", free_pgs[i]));
   3220 			}
   3221 #endif
   3222 			continue;
   3223 		}
   3224 
   3225 		mop = env->me_pghead;
   3226 		mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count;
   3227 
   3228 		/* Reserve records for me_pghead[]. Split it if multi-page,
   3229 		 * to avoid searching freeDB for a page range. Use keys in
   3230 		 * range [1,me_pglast]: Smaller than txnid of oldest reader.
   3231 		 */
   3232 		if (total_room >= mop_len) {
   3233 			if (total_room == mop_len || --more < 0)
   3234 				break;
   3235 		} else if (head_room >= maxfree_1pg && head_id > 1) {
   3236 			/* Keep current record (overflow page), add a new one */
   3237 			head_id--;
   3238 			head_room = 0;
   3239 		}
   3240 		/* (Re)write {key = head_id, IDL length = head_room} */
   3241 		total_room -= head_room;
   3242 		head_room = mop_len - total_room;
   3243 		if (head_room > maxfree_1pg && head_id > 1) {
   3244 			/* Overflow multi-page for part of me_pghead */
   3245 			head_room /= head_id; /* amortize page sizes */
   3246 			head_room += maxfree_1pg - head_room % (maxfree_1pg + 1);
   3247 		} else if (head_room < 0) {
   3248 			/* Rare case, not bothering to delete this record */
   3249 			head_room = 0;
   3250 		}
   3251 		key.mv_size = sizeof(head_id);
   3252 		key.mv_data = &head_id;
   3253 		data.mv_size = (head_room + 1) * sizeof(pgno_t);
   3254 		rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE);
   3255 		if (rc)
   3256 			return rc;
   3257 		/* IDL is initially empty, zero out at least the length */
   3258 		pgs = (pgno_t *)data.mv_data;
   3259 		j = head_room > clean_limit ? head_room : 0;
   3260 		do {
   3261 			pgs[j] = 0;
   3262 		} while (--j >= 0);
   3263 		total_room += head_room;
   3264 	}
   3265 
   3266 	/* Return loose page numbers to me_pghead, though usually none are
   3267 	 * left at this point.  The pages themselves remain in dirty_list.
   3268 	 */
   3269 	if (txn->mt_loose_pgs) {
   3270 		MDB_page *mp = txn->mt_loose_pgs;
   3271 		unsigned count = txn->mt_loose_count;
   3272 		MDB_IDL loose;
   3273 		/* Room for loose pages + temp IDL with same */
   3274 		if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0)
   3275 			return rc;
   3276 		mop = env->me_pghead;
   3277 		loose = mop + MDB_IDL_ALLOCLEN(mop) - count;
   3278 		for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp))
   3279 			loose[ ++count ] = mp->mp_pgno;
   3280 		loose[0] = count;
   3281 		mdb_midl_sort(loose);
   3282 		mdb_midl_xmerge(mop, loose);
   3283 		txn->mt_loose_pgs = NULL;
   3284 		txn->mt_loose_count = 0;
   3285 		mop_len = mop[0];
   3286 	}
   3287 
   3288 	/* Fill in the reserved me_pghead records */
   3289 	rc = MDB_SUCCESS;
   3290 	if (mop_len) {
   3291 		MDB_val key, data;
   3292 
   3293 		mop += mop_len;
   3294 		rc = mdb_cursor_first(&mc, &key, &data);
   3295 		for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) {
   3296 			txnid_t id = *(txnid_t *)key.mv_data;
   3297 			ssize_t	len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1;
   3298 			MDB_ID save;
   3299 
   3300 			mdb_tassert(txn, len >= 0 && id <= env->me_pglast);
   3301 			key.mv_data = &id;
   3302 			if (len > mop_len) {
   3303 				len = mop_len;
   3304 				data.mv_size = (len + 1) * sizeof(MDB_ID);
   3305 			}
   3306 			data.mv_data = mop -= len;
   3307 			save = mop[0];
   3308 			mop[0] = len;
   3309 			rc = mdb_cursor_put(&mc, &key, &data, MDB_CURRENT);
   3310 			mop[0] = save;
   3311 			if (rc || !(mop_len -= len))
   3312 				break;
   3313 		}
   3314 	}
   3315 	return rc;
   3316 }
   3317 
   3318 /** Flush (some) dirty pages to the map, after clearing their dirty flag.
   3319  * @param[in] txn the transaction that's being committed
   3320  * @param[in] keep number of initial pages in dirty_list to keep dirty.
   3321  * @return 0 on success, non-zero on failure.
   3322  */
   3323 static int
   3324 mdb_page_flush(MDB_txn *txn, int keep)
   3325 {
   3326 	MDB_env		*env = txn->mt_env;
   3327 	MDB_ID2L	dl = txn->mt_u.dirty_list;
   3328 	unsigned	psize = env->me_psize, j;
   3329 	int			i, pagecount = dl[0].mid, rc;
   3330 	size_t		size = 0, pos = 0;
   3331 	pgno_t		pgno = 0;
   3332 	MDB_page	*dp = NULL;
   3333 #ifdef _WIN32
   3334 	OVERLAPPED	ov;
   3335 #else
   3336 	struct iovec iov[MDB_COMMIT_PAGES];
   3337 	ssize_t		wpos = 0, wsize = 0, wres;
   3338 	size_t		next_pos = 1; /* impossible pos, so pos != next_pos */
   3339 	int			n = 0;
   3340 #endif
   3341 
   3342 	j = i = keep;
   3343 
   3344 	if (env->me_flags & MDB_WRITEMAP) {
   3345 		/* Clear dirty flags */
   3346 		while (++i <= pagecount) {
   3347 			dp = dl[i].mptr;
   3348 			/* Don't flush this page yet */
   3349 			if (dp->mp_flags & (P_LOOSE|P_KEEP)) {
   3350 				dp->mp_flags &= ~P_KEEP;
   3351 				dl[++j] = dl[i];
   3352 				continue;
   3353 			}
   3354 			dp->mp_flags &= ~P_DIRTY;
   3355 		}
   3356 		goto done;
   3357 	}
   3358 
   3359 	/* Write the pages */
   3360 	for (;;) {
   3361 		if (++i <= pagecount) {
   3362 			dp = dl[i].mptr;
   3363 			/* Don't flush this page yet */
   3364 			if (dp->mp_flags & (P_LOOSE|P_KEEP)) {
   3365 				dp->mp_flags &= ~P_KEEP;
   3366 				dl[i].mid = 0;
   3367 				continue;
   3368 			}
   3369 			pgno = dl[i].mid;
   3370 			/* clear dirty flag */
   3371 			dp->mp_flags &= ~P_DIRTY;
   3372 			pos = pgno * psize;
   3373 			size = psize;
   3374 			if (IS_OVERFLOW(dp)) size *= dp->mp_pages;
   3375 		}
   3376 #ifdef _WIN32
   3377 		else break;
   3378 
   3379 		/* Windows actually supports scatter/gather I/O, but only on
   3380 		 * unbuffered file handles. Since we're relying on the OS page
   3381 		 * cache for all our data, that's self-defeating. So we just
   3382 		 * write pages one at a time. We use the ov structure to set
   3383 		 * the write offset, to at least save the overhead of a Seek
   3384 		 * system call.
   3385 		 */
   3386 		DPRINTF(("committing page %"Z"u", pgno));
   3387 		memset(&ov, 0, sizeof(ov));
   3388 		ov.Offset = pos & 0xffffffff;
   3389 		ov.OffsetHigh = pos >> 16 >> 16;
   3390 		if (!WriteFile(env->me_fd, dp, size, NULL, &ov)) {
   3391 			rc = ErrCode();
   3392 			DPRINTF(("WriteFile: %d", rc));
   3393 			return rc;
   3394 		}
   3395 #else
   3396 		/* Write up to MDB_COMMIT_PAGES dirty pages at a time. */
   3397 		if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) {
   3398 			if (n) {
   3399 retry_write:
   3400 				/* Write previous page(s) */
   3401 #ifdef MDB_USE_PWRITEV
   3402 				wres = pwritev(env->me_fd, iov, n, wpos);
   3403 #else
   3404 				if (n == 1) {
   3405 					wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos);
   3406 				} else {
   3407 retry_seek:
   3408 					if (lseek(env->me_fd, wpos, SEEK_SET) == -1) {
   3409 						rc = ErrCode();
   3410 						if (rc == EINTR)
   3411 							goto retry_seek;
   3412 						DPRINTF(("lseek: %s", strerror(rc)));
   3413 						return rc;
   3414 					}
   3415 					wres = writev(env->me_fd, iov, n);
   3416 				}
   3417 #endif
   3418 				if (wres != wsize) {
   3419 					if (wres < 0) {
   3420 						rc = ErrCode();
   3421 						if (rc == EINTR)
   3422 							goto retry_write;
   3423 						DPRINTF(("Write error: %s", strerror(rc)));
   3424 					} else {
   3425 						rc = EIO; /* TODO: Use which error code? */
   3426 						DPUTS("short write, filesystem full?");
   3427 					}
   3428 					return rc;
   3429 				}
   3430 				n = 0;
   3431 			}
   3432 			if (i > pagecount)
   3433 				break;
   3434 			wpos = pos;
   3435 			wsize = 0;
   3436 		}
   3437 		DPRINTF(("committing page %"Z"u", pgno));
   3438 		next_pos = pos + size;
   3439 		iov[n].iov_len = size;
   3440 		iov[n].iov_base = (char *)dp;
   3441 		wsize += size;
   3442 		n++;
   3443 #endif	/* _WIN32 */
   3444 	}
   3445 
   3446 	/* MIPS has cache coherency issues, this is a no-op everywhere else
   3447 	 * Note: for any size >= on-chip cache size, entire on-chip cache is
   3448 	 * flushed.
   3449 	 */
   3450 	CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE);
   3451 
   3452 	for (i = keep; ++i <= pagecount; ) {
   3453 		dp = dl[i].mptr;
   3454 		/* This is a page we skipped above */
   3455 		if (!dl[i].mid) {
   3456 			dl[++j] = dl[i];
   3457 			dl[j].mid = dp->mp_pgno;
   3458 			continue;
   3459 		}
   3460 		mdb_dpage_free(env, dp);
   3461 	}
   3462 
   3463 done:
   3464 	i--;
   3465 	txn->mt_dirty_room += i - j;
   3466 	dl[0].mid = j;
   3467 	return MDB_SUCCESS;
   3468 }
   3469 
   3470 int
   3471 mdb_txn_commit(MDB_txn *txn)
   3472 {
   3473 	int		rc;
   3474 	unsigned int i, end_mode;
   3475 	MDB_env	*env;
   3476 
   3477 	if (txn == NULL)
   3478 		return EINVAL;
   3479 
   3480 	/* mdb_txn_end() mode for a commit which writes nothing */
   3481 	end_mode = MDB_END_EMPTY_COMMIT|MDB_END_UPDATE|MDB_END_SLOT|MDB_END_FREE;
   3482 
   3483 	if (txn->mt_child) {
   3484 		rc = mdb_txn_commit(txn->mt_child);
   3485 		if (rc)
   3486 			goto fail;
   3487 	}
   3488 
   3489 	env = txn->mt_env;
   3490 
   3491 	if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) {
   3492 		goto done;
   3493 	}
   3494 
   3495 	if (txn->mt_flags & (MDB_TXN_FINISHED|MDB_TXN_ERROR)) {
   3496 		DPUTS("txn has failed/finished, can't commit");
   3497 		if (txn->mt_parent)
   3498 			txn->mt_parent->mt_flags |= MDB_TXN_ERROR;
   3499 		rc = MDB_BAD_TXN;
   3500 		goto fail;
   3501 	}
   3502 
   3503 	if (txn->mt_parent) {
   3504 		MDB_txn *parent = txn->mt_parent;
   3505 		MDB_page **lp;
   3506 		MDB_ID2L dst, src;
   3507 		MDB_IDL pspill;
   3508 		unsigned x, y, len, ps_len;
   3509 
   3510 		/* Append our free list to parent's */
   3511 		rc = mdb_midl_append_list(&parent->mt_free_pgs, txn->mt_free_pgs);
   3512 		if (rc)
   3513 			goto fail;
   3514 		mdb_midl_free(txn->mt_free_pgs);
   3515 		/* Failures after this must either undo the changes
   3516 		 * to the parent or set MDB_TXN_ERROR in the parent.
   3517 		 */
   3518 
   3519 		parent->mt_next_pgno = txn->mt_next_pgno;
   3520 		parent->mt_flags = txn->mt_flags;
   3521 
   3522 		/* Merge our cursors into parent's and close them */
   3523 		mdb_cursors_close(txn, 1);
   3524 
   3525 		/* Update parent's DB table. */
   3526 		memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDB_db));
   3527 		parent->mt_numdbs = txn->mt_numdbs;
   3528 		parent->mt_dbflags[FREE_DBI] = txn->mt_dbflags[FREE_DBI];
   3529 		parent->mt_dbflags[MAIN_DBI] = txn->mt_dbflags[MAIN_DBI];
   3530 		for (i=CORE_DBS; i<txn->mt_numdbs; i++) {
   3531 			/* preserve parent's DB_NEW status */
   3532 			x = parent->mt_dbflags[i] & DB_NEW;
   3533 			parent->mt_dbflags[i] = txn->mt_dbflags[i] | x;
   3534 		}
   3535 
   3536 		dst = parent->mt_u.dirty_list;
   3537 		src = txn->mt_u.dirty_list;
   3538 		/* Remove anything in our dirty list from parent's spill list */
   3539 		if ((pspill = parent->mt_spill_pgs) && (ps_len = pspill[0])) {
   3540 			x = y = ps_len;
   3541 			pspill[0] = (pgno_t)-1;
   3542 			/* Mark our dirty pages as deleted in parent spill list */
   3543 			for (i=0, len=src[0].mid; ++i <= len; ) {
   3544 				MDB_ID pn = src[i].mid << 1;
   3545 				while (pn > pspill[x])
   3546 					x--;
   3547 				if (pn == pspill[x]) {
   3548 					pspill[x] = 1;
   3549 					y = --x;
   3550 				}
   3551 			}
   3552 			/* Squash deleted pagenums if we deleted any */
   3553 			for (x=y; ++x <= ps_len; )
   3554 				if (!(pspill[x] & 1))
   3555 					pspill[++y] = pspill[x];
   3556 			pspill[0] = y;
   3557 		}
   3558 
   3559 		/* Remove anything in our spill list from parent's dirty list */
   3560 		if (txn->mt_spill_pgs && txn->mt_spill_pgs[0]) {
   3561 			for (i=1; i<=txn->mt_spill_pgs[0]; i++) {
   3562 				MDB_ID pn = txn->mt_spill_pgs[i];
   3563 				if (pn & 1)
   3564 					continue;	/* deleted spillpg */
   3565 				pn >>= 1;
   3566 				y = mdb_mid2l_search(dst, pn);
   3567 				if (y <= dst[0].mid && dst[y].mid == pn) {
   3568 					free(dst[y].mptr);
   3569 					while (y < dst[0].mid) {
   3570 						dst[y] = dst[y+1];
   3571 						y++;
   3572 					}
   3573 					dst[0].mid--;
   3574 				}
   3575 			}
   3576 		}
   3577 
   3578 		/* Find len = length of merging our dirty list with parent's */
   3579 		x = dst[0].mid;
   3580 		dst[0].mid = 0;		/* simplify loops */
   3581 		if (parent->mt_parent) {
   3582 			len = x + src[0].mid;
   3583 			y = mdb_mid2l_search(src, dst[x].mid + 1) - 1;
   3584 			for (i = x; y && i; y--) {
   3585 				pgno_t yp = src[y].mid;
   3586 				while (yp < dst[i].mid)
   3587 					i--;
   3588 				if (yp == dst[i].mid) {
   3589 					i--;
   3590 					len--;
   3591 				}
   3592 			}
   3593 		} else { /* Simplify the above for single-ancestor case */
   3594 			len = MDB_IDL_UM_MAX - txn->mt_dirty_room;
   3595 		}
   3596 		/* Merge our dirty list with parent's */
   3597 		y = src[0].mid;
   3598 		for (i = len; y; dst[i--] = src[y--]) {
   3599 			pgno_t yp = src[y].mid;
   3600 			while (yp < dst[x].mid)
   3601 				dst[i--] = dst[x--];
   3602 			if (yp == dst[x].mid)
   3603 				free(dst[x--].mptr);
   3604 		}
   3605 		mdb_tassert(txn, i == x);
   3606 		dst[0].mid = len;
   3607 		free(txn->mt_u.dirty_list);
   3608 		parent->mt_dirty_room = txn->mt_dirty_room;
   3609 		if (txn->mt_spill_pgs) {
   3610 			if (parent->mt_spill_pgs) {
   3611 				/* TODO: Prevent failure here, so parent does not fail */
   3612 				rc = mdb_midl_append_list(&parent->mt_spill_pgs, txn->mt_spill_pgs);
   3613 				if (rc)
   3614 					parent->mt_flags |= MDB_TXN_ERROR;
   3615 				mdb_midl_free(txn->mt_spill_pgs);
   3616 				mdb_midl_sort(parent->mt_spill_pgs);
   3617 			} else {
   3618 				parent->mt_spill_pgs = txn->mt_spill_pgs;
   3619 			}
   3620 		}
   3621 
   3622 		/* Append our loose page list to parent's */
   3623 		for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(*lp))
   3624 			;
   3625 		*lp = txn->mt_loose_pgs;
   3626 		parent->mt_loose_count += txn->mt_loose_count;
   3627 
   3628 		parent->mt_child = NULL;
   3629 		mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead);
   3630 		free(txn);
   3631 		return rc;
   3632 	}
   3633 
   3634 	if (txn != env->me_txn) {
   3635 		DPUTS("attempt to commit unknown transaction");
   3636 		rc = EINVAL;
   3637 		goto fail;
   3638 	}
   3639 
   3640 	mdb_cursors_close(txn, 0);
   3641 
   3642 	if (!txn->mt_u.dirty_list[0].mid &&
   3643 		!(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS)))
   3644 		goto done;
   3645 
   3646 	DPRINTF(("committing txn %"Z"u %p on mdbenv %p, root page %"Z"u",
   3647 	    txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root));
   3648 
   3649 	/* Update DB root pointers */
   3650 	if (txn->mt_numdbs > CORE_DBS) {
   3651 		MDB_cursor mc;
   3652 		MDB_dbi i;
   3653 		MDB_val data;
   3654 		data.mv_size = sizeof(MDB_db);
   3655 
   3656 		mdb_cursor_init(&mc, txn, MAIN_DBI, NULL);
   3657 		for (i = CORE_DBS; i < txn->mt_numdbs; i++) {
   3658 			if (txn->mt_dbflags[i] & DB_DIRTY) {
   3659 				if (TXN_DBI_CHANGED(txn, i)) {
   3660 					rc = MDB_BAD_DBI;
   3661 					goto fail;
   3662 				}
   3663 				data.mv_data = &txn->mt_dbs[i];
   3664 				rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data,
   3665 					F_SUBDATA);
   3666 				if (rc)
   3667 					goto fail;
   3668 			}
   3669 		}
   3670 	}
   3671 
   3672 	rc = mdb_freelist_save(txn);
   3673 	if (rc)
   3674 		goto fail;
   3675 
   3676 	mdb_midl_free(env->me_pghead);
   3677 	env->me_pghead = NULL;
   3678 	mdb_midl_shrink(&txn->mt_free_pgs);
   3679 
   3680 #if (MDB_DEBUG) > 2
   3681 	mdb_audit(txn);
   3682 #endif
   3683 
   3684 	if ((rc = mdb_page_flush(txn, 0)) ||
   3685 		(rc = mdb_env_sync(env, 0)) ||
   3686 		(rc = mdb_env_write_meta(txn)))
   3687 		goto fail;
   3688 	end_mode = MDB_END_COMMITTED|MDB_END_UPDATE;
   3689 
   3690 done:
   3691 	mdb_txn_end(txn, end_mode);
   3692 	return MDB_SUCCESS;
   3693 
   3694 fail:
   3695 	mdb_txn_abort(txn);
   3696 	return rc;
   3697 }
   3698 
   3699 /** Read the environment parameters of a DB environment before
   3700  * mapping it into memory.
   3701  * @param[in] env the environment handle
   3702  * @param[out] meta address of where to store the meta information
   3703  * @return 0 on success, non-zero on failure.
   3704  */
   3705 static int ESECT
   3706 mdb_env_read_header(MDB_env *env, MDB_meta *meta)
   3707 {
   3708 	MDB_metabuf	pbuf;
   3709 	MDB_page	*p;
   3710 	MDB_meta	*m;
   3711 	int			i, rc, off;
   3712 	enum { Size = sizeof(pbuf) };
   3713 
   3714 	/* We don't know the page size yet, so use a minimum value.
   3715 	 * Read both meta pages so we can use the latest one.
   3716 	 */
   3717 
   3718 	for (i=off=0; i<NUM_METAS; i++, off += meta->mm_psize) {
   3719 #ifdef _WIN32
   3720 		DWORD len;
   3721 		OVERLAPPED ov;
   3722 		memset(&ov, 0, sizeof(ov));
   3723 		ov.Offset = off;
   3724 		rc = ReadFile(env->me_fd, &pbuf, Size, &len, &ov) ? (int)len : -1;
   3725 		if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF)
   3726 			rc = 0;
   3727 #else
   3728 		rc = pread(env->me_fd, &pbuf, Size, off);
   3729 #endif
   3730 		if (rc != Size) {
   3731 			if (rc == 0 && off == 0)
   3732 				return ENOENT;
   3733 			rc = rc < 0 ? (int) ErrCode() : MDB_INVALID;
   3734 			DPRINTF(("read: %s", mdb_strerror(rc)));
   3735 			return rc;
   3736 		}
   3737 
   3738 		p = (MDB_page *)&pbuf;
   3739 
   3740 		if (!F_ISSET(p->mp_flags, P_META)) {
   3741 			DPRINTF(("page %"Z"u not a meta page", p->mp_pgno));
   3742 			return MDB_INVALID;
   3743 		}
   3744 
   3745 		m = METADATA(p);
   3746 		if (m->mm_magic != MDB_MAGIC) {
   3747 			DPUTS("meta has invalid magic");
   3748 			return MDB_INVALID;
   3749 		}
   3750 
   3751 		if (m->mm_version != MDB_DATA_VERSION) {
   3752 			DPRINTF(("database is version %u, expected version %u",
   3753 				m->mm_version, MDB_DATA_VERSION));
   3754 			return MDB_VERSION_MISMATCH;
   3755 		}
   3756 
   3757 		if (off == 0 || m->mm_txnid > meta->mm_txnid)
   3758 			*meta = *m;
   3759 	}
   3760 	return 0;
   3761 }
   3762 
   3763 /** Fill in most of the zeroed #MDB_meta for an empty database environment */
   3764 static void ESECT
   3765 mdb_env_init_meta0(MDB_env *env, MDB_meta *meta)
   3766 {
   3767 	meta->mm_magic = MDB_MAGIC;
   3768 	meta->mm_version = MDB_DATA_VERSION;
   3769 	meta->mm_mapsize = env->me_mapsize;
   3770 	meta->mm_psize = env->me_psize;
   3771 	meta->mm_last_pg = NUM_METAS-1;
   3772 	meta->mm_flags = env->me_flags & 0xffff;
   3773 	meta->mm_flags |= MDB_INTEGERKEY; /* this is mm_dbs[FREE_DBI].md_flags */
   3774 	meta->mm_dbs[FREE_DBI].md_root = P_INVALID;
   3775 	meta->mm_dbs[MAIN_DBI].md_root = P_INVALID;
   3776 }
   3777 
   3778 /** Write the environment parameters of a freshly created DB environment.
   3779  * @param[in] env the environment handle
   3780  * @param[in] meta the #MDB_meta to write
   3781  * @return 0 on success, non-zero on failure.
   3782  */
   3783 static int ESECT
   3784 mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
   3785 {
   3786 	MDB_page *p, *q;
   3787 	int rc;
   3788 	unsigned int	 psize;
   3789 #ifdef _WIN32
   3790 	DWORD len;
   3791 	OVERLAPPED ov;
   3792 	memset(&ov, 0, sizeof(ov));
   3793 #define DO_PWRITE(rc, fd, ptr, size, len, pos)	do { \
   3794 	ov.Offset = pos;	\
   3795 	rc = WriteFile(fd, ptr, size, &len, &ov);	} while(0)
   3796 #else
   3797 	int len;
   3798 #define DO_PWRITE(rc, fd, ptr, size, len, pos)	do { \
   3799 	len = pwrite(fd, ptr, size, pos);	\
   3800 	if (len == -1 && ErrCode() == EINTR) continue; \
   3801 	rc = (len >= 0); break; } while(1)
   3802 #endif
   3803 
   3804 	DPUTS("writing new meta page");
   3805 
   3806 	psize = env->me_psize;
   3807 
   3808 	p = calloc(NUM_METAS, psize);
   3809 	if (!p)
   3810 		return ENOMEM;
   3811 
   3812 	p->mp_pgno = 0;
   3813 	p->mp_flags = P_META;
   3814 	*(MDB_meta *)METADATA(p) = *meta;
   3815 
   3816 	q = (MDB_page *)((char *)p + psize);
   3817 	q->mp_pgno = 1;
   3818 	q->mp_flags = P_META;
   3819 	*(MDB_meta *)METADATA(q) = *meta;
   3820 
   3821 	DO_PWRITE(rc, env->me_fd, p, psize * NUM_METAS, len, 0);
   3822 	if (!rc)
   3823 		rc = ErrCode();
   3824 	else if ((unsigned) len == psize * NUM_METAS)
   3825 		rc = MDB_SUCCESS;
   3826 	else
   3827 		rc = ENOSPC;
   3828 	free(p);
   3829 	return rc;
   3830 }
   3831 
   3832 /** Update the environment info to commit a transaction.
   3833  * @param[in] txn the transaction that's being committed
   3834  * @return 0 on success, non-zero on failure.
   3835  */
   3836 static int
   3837 mdb_env_write_meta(MDB_txn *txn)
   3838 {
   3839 	MDB_env *env;
   3840 	MDB_meta	meta, metab, *mp;
   3841 	unsigned flags;
   3842 	size_t mapsize;
   3843 	off_t off;
   3844 	int rc, len, toggle;
   3845 	char *ptr;
   3846 	HANDLE mfd;
   3847 #ifdef _WIN32
   3848 	OVERLAPPED ov;
   3849 #else
   3850 	int r2;
   3851 #endif
   3852 
   3853 	toggle = txn->mt_txnid & 1;
   3854 	DPRINTF(("writing meta page %d for root page %"Z"u",
   3855 		toggle, txn->mt_dbs[MAIN_DBI].md_root));
   3856 
   3857 	env = txn->mt_env;
   3858 	flags = env->me_flags;
   3859 	mp = env->me_metas[toggle];
   3860 	mapsize = env->me_metas[toggle ^ 1]->mm_mapsize;
   3861 	/* Persist any increases of mapsize config */
   3862 	if (mapsize < env->me_mapsize)
   3863 		mapsize = env->me_mapsize;
   3864 
   3865 	if (flags & MDB_WRITEMAP) {
   3866 		mp->mm_mapsize = mapsize;
   3867 		mp->mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI];
   3868 		mp->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI];
   3869 		mp->mm_last_pg = txn->mt_next_pgno - 1;
   3870 #if (__GNUC__ * 100 + __GNUC_MINOR__ >= 404) && /* TODO: portability */	\
   3871 	!(defined(__i386__) || defined(__x86_64__))
   3872 		/* LY: issue a memory barrier, if not x86. ITS#7969 */
   3873 		__sync_synchronize();
   3874 #endif
   3875 		mp->mm_txnid = txn->mt_txnid;
   3876 		if (!(flags & (MDB_NOMETASYNC|MDB_NOSYNC))) {
   3877 			unsigned meta_size = env->me_psize;
   3878 			rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC;
   3879 			ptr = (char *)mp - PAGEHDRSZ;
   3880 #ifndef _WIN32	/* POSIX msync() requires ptr = start of OS page */
   3881 			r2 = (ptr - env->me_map) & (env->me_os_psize - 1);
   3882 			ptr -= r2;
   3883 			meta_size += r2;
   3884 #endif
   3885 			if (MDB_MSYNC(ptr, meta_size, rc)) {
   3886 				rc = ErrCode();
   3887 				goto fail;
   3888 			}
   3889 		}
   3890 		goto done;
   3891 	}
   3892 	metab.mm_txnid = mp->mm_txnid;
   3893 	metab.mm_last_pg = mp->mm_last_pg;
   3894 
   3895 	meta.mm_mapsize = mapsize;
   3896 	meta.mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI];
   3897 	meta.mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI];
   3898 	meta.mm_last_pg = txn->mt_next_pgno - 1;
   3899 	meta.mm_txnid = txn->mt_txnid;
   3900 
   3901 	off = offsetof(MDB_meta, mm_mapsize);
   3902 	ptr = (char *)&meta + off;
   3903 	len = sizeof(MDB_meta) - off;
   3904 	off += (char *)mp - env->me_map;
   3905 
   3906 	/* Write to the SYNC fd unless MDB_NOSYNC/MDB_NOMETASYNC.
   3907 	 * (me_mfd goes to the same file as me_fd, but writing to it
   3908 	 * also syncs to disk.  Avoids a separate fdatasync() call.)
   3909 	 */
   3910 	mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd;
   3911 #ifdef _WIN32
   3912 	{
   3913 		memset(&ov, 0, sizeof(ov));
   3914 		ov.Offset = off;
   3915 		if (!WriteFile(mfd, ptr, len, (DWORD *)&rc, &ov))
   3916 			rc = -1;
   3917 	}
   3918 #else
   3919 retry_write:
   3920 	rc = pwrite(mfd, ptr, len, off);
   3921 #endif
   3922 	if (rc != len) {
   3923 		rc = rc < 0 ? ErrCode() : EIO;
   3924 #ifndef _WIN32
   3925 		if (rc == EINTR)
   3926 			goto retry_write;
   3927 #endif
   3928 		DPUTS("write failed, disk error?");
   3929 		/* On a failure, the pagecache still contains the new data.
   3930 		 * Write some old data back, to prevent it from being used.
   3931 		 * Use the non-SYNC fd; we know it will fail anyway.
   3932 		 */
   3933 		meta.mm_last_pg = metab.mm_last_pg;
   3934 		meta.mm_txnid = metab.mm_txnid;
   3935 #ifdef _WIN32
   3936 		memset(&ov, 0, sizeof(ov));
   3937 		ov.Offset = off;
   3938 		WriteFile(env->me_fd, ptr, len, NULL, &ov);
   3939 #else
   3940 		r2 = pwrite(env->me_fd, ptr, len, off);
   3941 		(void)r2;	/* Silence warnings. We don't care about pwrite's return value */
   3942 #endif
   3943 fail:
   3944 		env->me_flags |= MDB_FATAL_ERROR;
   3945 		return rc;
   3946 	}
   3947 	/* MIPS has cache coherency issues, this is a no-op everywhere else */
   3948 	CACHEFLUSH(env->me_map + off, len, DCACHE);
   3949 done:
   3950 	/* Memory ordering issues are irrelevant; since the entire writer
   3951 	 * is wrapped by wmutex, all of these changes will become visible
   3952 	 * after the wmutex is unlocked. Since the DB is multi-version,
   3953 	 * readers will get consistent data regardless of how fresh or
   3954 	 * how stale their view of these values is.
   3955 	 */
   3956 	if (env->me_txns)
   3957 		env->me_txns->mti_txnid = txn->mt_txnid;
   3958 
   3959 	return MDB_SUCCESS;
   3960 }
   3961 
   3962 /** Check both meta pages to see which one is newer.
   3963  * @param[in] env the environment handle
   3964  * @return newest #MDB_meta.
   3965  */
   3966 static MDB_meta *
   3967 mdb_env_pick_meta(const MDB_env *env)
   3968 {
   3969 	MDB_meta *const *metas = env->me_metas;
   3970 	return metas[ metas[0]->mm_txnid < metas[1]->mm_txnid ];
   3971 }
   3972 
   3973 int ESECT
   3974 mdb_env_create(MDB_env **env)
   3975 {
   3976 	MDB_env *e;
   3977 
   3978 	e = calloc(1, sizeof(MDB_env));
   3979 	if (!e)
   3980 		return ENOMEM;
   3981 
   3982 	e->me_maxreaders = DEFAULT_READERS;
   3983 	e->me_maxdbs = e->me_numdbs = CORE_DBS;
   3984 	e->me_fd = INVALID_HANDLE_VALUE;
   3985 	e->me_lfd = INVALID_HANDLE_VALUE;
   3986 	e->me_mfd = INVALID_HANDLE_VALUE;
   3987 #ifdef MDB_USE_POSIX_SEM
   3988 	e->me_rmutex = SEM_FAILED;
   3989 	e->me_wmutex = SEM_FAILED;
   3990 #endif
   3991 	e->me_pid = getpid();
   3992 	GET_PAGESIZE(e->me_os_psize);
   3993 	VGMEMP_CREATE(e,0,0);
   3994 	*env = e;
   3995 	return MDB_SUCCESS;
   3996 }
   3997 
   3998 static int ESECT
   3999 mdb_env_map(MDB_env *env, void *addr)
   4000 {
   4001 	MDB_page *p;
   4002 	unsigned int flags = env->me_flags;
   4003 #ifdef _WIN32
   4004 	int rc;
   4005 	HANDLE mh;
   4006 	LONG sizelo, sizehi;
   4007 	size_t msize;
   4008 
   4009 	if (flags & MDB_RDONLY) {
   4010 		/* Don't set explicit map size, use whatever exists */
   4011 		msize = 0;
   4012 		sizelo = 0;
   4013 		sizehi = 0;
   4014 	} else {
   4015 		msize = env->me_mapsize;
   4016 		sizelo = msize & 0xffffffff;
   4017 		sizehi = msize >> 16 >> 16; /* only needed on Win64 */
   4018 
   4019 		/* Windows won't create mappings for zero length files.
   4020 		 * and won't map more than the file size.
   4021 		 * Just set the maxsize right now.
   4022 		 */
   4023 		if (!(flags & MDB_WRITEMAP) && (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo
   4024 			|| !SetEndOfFile(env->me_fd)
   4025 			|| SetFilePointer(env->me_fd, 0, NULL, 0) != 0))
   4026 			return ErrCode();
   4027 	}
   4028 
   4029 	mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ?
   4030 		PAGE_READWRITE : PAGE_READONLY,
   4031 		sizehi, sizelo, NULL);
   4032 	if (!mh)
   4033 		return ErrCode();
   4034 	env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ?
   4035 		FILE_MAP_WRITE : FILE_MAP_READ,
   4036 		0, 0, msize, addr);
   4037 	rc = env->me_map ? 0 : ErrCode();
   4038 	CloseHandle(mh);
   4039 	if (rc)
   4040 		return rc;
   4041 #else
   4042 	int mmap_flags = MAP_SHARED;
   4043 	int prot = PROT_READ;
   4044 #ifdef MAP_NOSYNC	/* Used on FreeBSD */
   4045 	if (flags & MDB_NOSYNC)
   4046 		mmap_flags |= MAP_NOSYNC;
   4047 #endif
   4048 	if (flags & MDB_WRITEMAP) {
   4049 		prot |= PROT_WRITE;
   4050 		if (ftruncate(env->me_fd, env->me_mapsize) < 0)
   4051 			return ErrCode();
   4052 	}
   4053 	env->me_map = mmap(addr, env->me_mapsize, prot, mmap_flags,
   4054 		env->me_fd, 0);
   4055 	if (env->me_map == MAP_FAILED) {
   4056 		env->me_map = NULL;
   4057 		return ErrCode();
   4058 	}
   4059 
   4060 	if (flags & MDB_NORDAHEAD) {
   4061 		/* Turn off readahead. It's harmful when the DB is larger than RAM. */
   4062 #ifdef MADV_RANDOM
   4063 		madvise(env->me_map, env->me_mapsize, MADV_RANDOM);
   4064 #else
   4065 #ifdef POSIX_MADV_RANDOM
   4066 		posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM);
   4067 #endif /* POSIX_MADV_RANDOM */
   4068 #endif /* MADV_RANDOM */
   4069 	}
   4070 #endif /* _WIN32 */
   4071 
   4072 	/* Can happen because the address argument to mmap() is just a
   4073 	 * hint.  mmap() can pick another, e.g. if the range is in use.
   4074 	 * The MAP_FIXED flag would prevent that, but then mmap could
   4075 	 * instead unmap existing pages to make room for the new map.
   4076 	 */
   4077 	if (addr && env->me_map != addr)
   4078 		return EBUSY;	/* TODO: Make a new MDB_* error code? */
   4079 
   4080 	p = (MDB_page *)env->me_map;
   4081 	env->me_metas[0] = METADATA(p);
   4082 	env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + env->me_psize);
   4083 
   4084 	return MDB_SUCCESS;
   4085 }
   4086 
   4087 int ESECT
   4088 mdb_env_set_mapsize(MDB_env *env, size_t size)
   4089 {
   4090 	/* If env is already open, caller is responsible for making
   4091 	 * sure there are no active txns.
   4092 	 */
   4093 	if (env->me_map) {
   4094 		int rc;
   4095 		MDB_meta *meta;
   4096 		void *old;
   4097 		if (env->me_txn)
   4098 			return EINVAL;
   4099 		meta = mdb_env_pick_meta(env);
   4100 		if (!size)
   4101 			size = meta->mm_mapsize;
   4102 		{
   4103 			/* Silently round up to minimum if the size is too small */
   4104 			size_t minsize = (meta->mm_last_pg + 1) * env->me_psize;
   4105 			if (size < minsize)
   4106 				size = minsize;
   4107 		}
   4108 		munmap(env->me_map, env->me_mapsize);
   4109 		env->me_mapsize = size;
   4110 		old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL;
   4111 		rc = mdb_env_map(env, old);
   4112 		if (rc)
   4113 			return rc;
   4114 	}
   4115 	env->me_mapsize = size;
   4116 	if (env->me_psize)
   4117 		env->me_maxpg = env->me_mapsize / env->me_psize;
   4118 	return MDB_SUCCESS;
   4119 }
   4120 
   4121 int ESECT
   4122 mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs)
   4123 {
   4124 	if (env->me_map)
   4125 		return EINVAL;
   4126 	env->me_maxdbs = dbs + CORE_DBS;
   4127 	return MDB_SUCCESS;
   4128 }
   4129 
   4130 int ESECT
   4131 mdb_env_set_maxreaders(MDB_env *env, unsigned int readers)
   4132 {
   4133 	if (env->me_map || readers < 1)
   4134 		return EINVAL;
   4135 	env->me_maxreaders = readers;
   4136 	return MDB_SUCCESS;
   4137 }
   4138 
   4139 int ESECT
   4140 mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers)
   4141 {
   4142 	if (!env || !readers)
   4143 		return EINVAL;
   4144 	*readers = env->me_maxreaders;
   4145 	return MDB_SUCCESS;
   4146 }
   4147 
   4148 static int ESECT
   4149 mdb_fsize(HANDLE fd, size_t *size)
   4150 {
   4151 #ifdef _WIN32
   4152 	LARGE_INTEGER fsize;
   4153 
   4154 	if (!GetFileSizeEx(fd, &fsize))
   4155 		return ErrCode();
   4156 
   4157 	*size = fsize.QuadPart;
   4158 #else
   4159 	struct stat st;
   4160 
   4161 	if (fstat(fd, &st))
   4162 		return ErrCode();
   4163 
   4164 	*size = st.st_size;
   4165 #endif
   4166 	return MDB_SUCCESS;
   4167 }
   4168 
   4169 
   4170 #ifdef _WIN32
   4171 typedef wchar_t	mdb_nchar_t;
   4172 # define MDB_NAME(str)	L##str
   4173 # define mdb_name_cpy	wcscpy
   4174 #else
   4175 /** Character type for file names: char on Unix, wchar_t on Windows */
   4176 typedef char	mdb_nchar_t;
   4177 # define MDB_NAME(str)	str		/**< #mdb_nchar_t[] string literal */
   4178 # define mdb_name_cpy	strcpy	/**< Copy name (#mdb_nchar_t string) */
   4179 #endif
   4180 
   4181 /** Filename - string of #mdb_nchar_t[] */
   4182 typedef struct MDB_name {
   4183 	int mn_len;					/**< Length  */
   4184 	int mn_alloced;				/**< True if #mn_val was malloced */
   4185 	mdb_nchar_t	*mn_val;		/**< Contents */
   4186 } MDB_name;
   4187 
   4188 /** Filename suffixes [datafile,lockfile][without,with MDB_NOSUBDIR] */
   4189 static const mdb_nchar_t *const mdb_suffixes[2][2] = {
   4190 	{ MDB_NAME("/data.mdb"), MDB_NAME("")      },
   4191 	{ MDB_NAME("/lock.mdb"), MDB_NAME("-lock") }
   4192 };
   4193 
   4194 #define MDB_SUFFLEN 9	/**< Max string length in #mdb_suffixes[] */
   4195 
   4196 /** Set up filename + scratch area for filename suffix, for opening files.
   4197  * It should be freed with #mdb_fname_destroy().
   4198  * On Windows, paths are converted from char *UTF-8 to wchar_t *UTF-16.
   4199  *
   4200  * @param[in] path Pathname for #mdb_env_open().
   4201  * @param[in] envflags Whether a subdir and/or lockfile will be used.
   4202  * @param[out] fname Resulting filename, with room for a suffix if necessary.
   4203  */
   4204 static int ESECT
   4205 mdb_fname_init(const char *path, unsigned envflags, MDB_name *fname)
   4206 {
   4207 	int no_suffix = F_ISSET(envflags, MDB_NOSUBDIR|MDB_NOLOCK);
   4208 	fname->mn_alloced = 0;
   4209 #ifdef _WIN32
   4210 	return utf8_to_utf16(path, fname, no_suffix ? 0 : MDB_SUFFLEN);
   4211 #else
   4212 	fname->mn_len = strlen(path);
   4213 	if (no_suffix)
   4214 		fname->mn_val = (char *) path;
   4215 	else if ((fname->mn_val = malloc(fname->mn_len + MDB_SUFFLEN+1)) != NULL) {
   4216 		fname->mn_alloced = 1;
   4217 		strcpy(fname->mn_val, path);
   4218 	}
   4219 	else
   4220 		return ENOMEM;
   4221 	return MDB_SUCCESS;
   4222 #endif
   4223 }
   4224 
   4225 /** Destroy \b fname from #mdb_fname_init() */
   4226 #define mdb_fname_destroy(fname) \
   4227 	do { if ((fname).mn_alloced) free((fname).mn_val); } while (0)
   4228 
   4229 #ifdef O_CLOEXEC /* POSIX.1-2008: Set FD_CLOEXEC atomically at open() */
   4230 # define MDB_CLOEXEC		O_CLOEXEC
   4231 #else
   4232 # define MDB_CLOEXEC		0
   4233 #endif
   4234 
   4235 /** File type, access mode etc. for #mdb_fopen() */
   4236 enum mdb_fopen_type {
   4237 #ifdef _WIN32
   4238 	MDB_O_RDONLY, MDB_O_RDWR, MDB_O_META, MDB_O_COPY, MDB_O_LOCKS
   4239 #else
   4240 	/* A comment in mdb_fopen() explains some O_* flag choices. */
   4241 	MDB_O_RDONLY= O_RDONLY,                            /**< for RDONLY me_fd */
   4242 	MDB_O_RDWR  = O_RDWR  |O_CREAT,                    /**< for me_fd */
   4243 	MDB_O_META  = O_WRONLY|MDB_DSYNC     |MDB_CLOEXEC, /**< for me_mfd */
   4244 	MDB_O_COPY  = O_WRONLY|O_CREAT|O_EXCL|MDB_CLOEXEC, /**< for #mdb_env_copy() */
   4245 	/** Bitmask for open() flags in enum #mdb_fopen_type.  The other bits
   4246 	 * distinguish otherwise-equal MDB_O_* constants from each other.
   4247 	 */
   4248 	MDB_O_MASK  = MDB_O_RDWR|MDB_CLOEXEC | MDB_O_RDONLY|MDB_O_META|MDB_O_COPY,
   4249 	MDB_O_LOCKS = MDB_O_RDWR|MDB_CLOEXEC | ((MDB_O_MASK+1) & ~MDB_O_MASK) /**< for me_lfd */
   4250 #endif
   4251 };
   4252 
   4253 /** Open an LMDB file.
   4254  * @param[in] env	The LMDB environment.
   4255  * @param[in,out] fname	Path from from #mdb_fname_init().  A suffix is
   4256  * appended if necessary to create the filename, without changing mn_len.
   4257  * @param[in] which	Determines file type, access mode, etc.
   4258  * @param[in] mode	The Unix permissions for the file, if we create it.
   4259  * @param[out] res	Resulting file handle.
   4260  * @return 0 on success, non-zero on failure.
   4261  */
   4262 static int ESECT
   4263 mdb_fopen(const MDB_env *env, MDB_name *fname,
   4264 	enum mdb_fopen_type which, mdb_mode_t mode,
   4265 	HANDLE *res)
   4266 {
   4267 	int rc = MDB_SUCCESS;
   4268 	HANDLE fd;
   4269 #ifdef _WIN32
   4270 	DWORD acc, share, disp, attrs;
   4271 #else
   4272 	int flags;
   4273 #endif
   4274 
   4275 	if (fname->mn_alloced)		/* modifiable copy */
   4276 		mdb_name_cpy(fname->mn_val + fname->mn_len,
   4277 			mdb_suffixes[which==MDB_O_LOCKS][F_ISSET(env->me_flags, MDB_NOSUBDIR)]);
   4278 
   4279 	/* The directory must already exist.  Usually the file need not.
   4280 	 * MDB_O_META requires the file because we already created it using
   4281 	 * MDB_O_RDWR.  MDB_O_COPY must not overwrite an existing file.
   4282 	 *
   4283 	 * With MDB_O_COPY we do not want the OS to cache the writes, since
   4284 	 * the source data is already in the OS cache.
   4285 	 *
   4286 	 * The lockfile needs FD_CLOEXEC (close file descriptor on exec*())
   4287 	 * to avoid the flock() issues noted under Caveats in lmdb.h.
   4288 	 * Also set it for other filehandles which the user cannot get at
   4289 	 * and close himself, which he may need after fork().  I.e. all but
   4290 	 * me_fd, which programs do use via mdb_env_get_fd().
   4291 	 */
   4292 
   4293 #ifdef _WIN32
   4294 	acc = GENERIC_READ|GENERIC_WRITE;
   4295 	share = FILE_SHARE_READ|FILE_SHARE_WRITE;
   4296 	disp = OPEN_ALWAYS;
   4297 	attrs = FILE_ATTRIBUTE_NORMAL;
   4298 	switch (which) {
   4299 	case MDB_O_RDONLY:			/* read-only datafile */
   4300 		acc = GENERIC_READ;
   4301 		disp = OPEN_EXISTING;
   4302 		break;
   4303 	case MDB_O_META:			/* for writing metapages */
   4304 		acc = GENERIC_WRITE;
   4305 		disp = OPEN_EXISTING;
   4306 		attrs = FILE_ATTRIBUTE_NORMAL|FILE_FLAG_WRITE_THROUGH;
   4307 		break;
   4308 	case MDB_O_COPY:			/* mdb_env_copy() & co */
   4309 		acc = GENERIC_WRITE;
   4310 		share = 0;
   4311 		disp = CREATE_NEW;
   4312 		attrs = FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH;
   4313 		break;
   4314 	default: break;	/* silence gcc -Wswitch (not all enum values handled) */
   4315 	}
   4316 	fd = CreateFileW(fname->mn_val, acc, share, NULL, disp, attrs, NULL);
   4317 #else
   4318 	fd = open(fname->mn_val, which & MDB_O_MASK, mode);
   4319 #endif
   4320 
   4321 	if (fd == INVALID_HANDLE_VALUE)
   4322 		rc = ErrCode();
   4323 #ifndef _WIN32
   4324 	else {
   4325 		if (which != MDB_O_RDONLY && which != MDB_O_RDWR) {
   4326 			/* Set CLOEXEC if we could not pass it to open() */
   4327 			if (!MDB_CLOEXEC && (flags = fcntl(fd, F_GETFD)) != -1)
   4328 				(void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
   4329 		}
   4330 		if (which == MDB_O_COPY && env->me_psize >= env->me_os_psize) {
   4331 			/* This may require buffer alignment.  There is no portable
   4332 			 * way to ask how much, so we require OS pagesize alignment.
   4333 			 */
   4334 # ifdef F_NOCACHE	/* __APPLE__ */
   4335 			(void) fcntl(fd, F_NOCACHE, 1);
   4336 # elif defined O_DIRECT
   4337 			/* open(...O_DIRECT...) would break on filesystems without
   4338 			 * O_DIRECT support (ITS#7682). Try to set it here instead.
   4339 			 */
   4340 			if ((flags = fcntl(fd, F_GETFL)) != -1)
   4341 				(void) fcntl(fd, F_SETFL, flags | O_DIRECT);
   4342 # endif
   4343 		}
   4344 	}
   4345 #endif	/* !_WIN32 */
   4346 
   4347 	*res = fd;
   4348 	return rc;
   4349 }
   4350 
   4351 
   4352 #ifdef BROKEN_FDATASYNC
   4353 #include <sys/utsname.h>
   4354 #include <sys/vfs.h>
   4355 #endif
   4356 
   4357 /** Further setup required for opening an LMDB environment
   4358  */
   4359 static int ESECT
   4360 mdb_env_open2(MDB_env *env)
   4361 {
   4362 	unsigned int flags = env->me_flags;
   4363 	int i, newenv = 0, rc;
   4364 	MDB_meta meta;
   4365 
   4366 #ifdef _WIN32
   4367 	/* See if we should use QueryLimited */
   4368 	rc = GetVersion();
   4369 	if ((rc & 0xff) > 5)
   4370 		env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION;
   4371 	else
   4372 		env->me_pidquery = PROCESS_QUERY_INFORMATION;
   4373 #endif /* _WIN32 */
   4374 
   4375 #ifdef BROKEN_FDATASYNC
   4376 	/* ext3/ext4 fdatasync is broken on some older Linux kernels.
   4377 	 * https://lkml.org/lkml/2012/9/3/83
   4378 	 * Kernels after 3.6-rc6 are known good.
   4379 	 * https://lkml.org/lkml/2012/9/10/556
   4380 	 * See if the DB is on ext3/ext4, then check for new enough kernel
   4381 	 * Kernels 2.6.32.60, 2.6.34.15, 3.2.30, and 3.5.4 are also known
   4382 	 * to be patched.
   4383 	 */
   4384 	{
   4385 		struct statfs st;
   4386 		fstatfs(env->me_fd, &st);
   4387 		while (st.f_type == 0xEF53) {
   4388 			struct utsname uts;
   4389 			int i;
   4390 			uname(&uts);
   4391 			if (uts.release[0] < '3') {
   4392 				if (!strncmp(uts.release, "2.6.32.", 7)) {
   4393 					i = atoi(uts.release+7);
   4394 					if (i >= 60)
   4395 						break;	/* 2.6.32.60 and newer is OK */
   4396 				} else if (!strncmp(uts.release, "2.6.34.", 7)) {
   4397 					i = atoi(uts.release+7);
   4398 					if (i >= 15)
   4399 						break;	/* 2.6.34.15 and newer is OK */
   4400 				}
   4401 			} else if (uts.release[0] == '3') {
   4402 				i = atoi(uts.release+2);
   4403 				if (i > 5)
   4404 					break;	/* 3.6 and newer is OK */
   4405 				if (i == 5) {
   4406 					i = atoi(uts.release+4);
   4407 					if (i >= 4)
   4408 						break;	/* 3.5.4 and newer is OK */
   4409 				} else if (i == 2) {
   4410 					i = atoi(uts.release+4);
   4411 					if (i >= 30)
   4412 						break;	/* 3.2.30 and newer is OK */
   4413 				}
   4414 			} else {	/* 4.x and newer is OK */
   4415 				break;
   4416 			}
   4417 			env->me_flags |= MDB_FSYNCONLY;
   4418 			break;
   4419 		}
   4420 	}
   4421 #endif
   4422 
   4423 	if ((i = mdb_env_read_header(env, &meta)) != 0) {
   4424 		if (i != ENOENT)
   4425 			return i;
   4426 		DPUTS("new mdbenv");
   4427 		newenv = 1;
   4428 		env->me_psize = env->me_os_psize;
   4429 		if (env->me_psize > MAX_PAGESIZE)
   4430 			env->me_psize = MAX_PAGESIZE;
   4431 		memset(&meta, 0, sizeof(meta));
   4432 		mdb_env_init_meta0(env, &meta);
   4433 		meta.mm_mapsize = DEFAULT_MAPSIZE;
   4434 	} else {
   4435 		env->me_psize = meta.mm_psize;
   4436 	}
   4437 
   4438 	/* Was a mapsize configured? */
   4439 	if (!env->me_mapsize) {
   4440 		env->me_mapsize = meta.mm_mapsize;
   4441 	}
   4442 	{
   4443 		/* Make sure mapsize >= committed data size.  Even when using
   4444 		 * mm_mapsize, which could be broken in old files (ITS#7789).
   4445 		 */
   4446 		size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize;
   4447 		if (env->me_mapsize < minsize)
   4448 			env->me_mapsize = minsize;
   4449 	}
   4450 	meta.mm_mapsize = env->me_mapsize;
   4451 
   4452 	if (newenv && !(flags & MDB_FIXEDMAP)) {
   4453 		/* mdb_env_map() may grow the datafile.  Write the metapages
   4454 		 * first, so the file will be valid if initialization fails.
   4455 		 * Except with FIXEDMAP, since we do not yet know mm_address.
   4456 		 * We could fill in mm_address later, but then a different
   4457 		 * program might end up doing that - one with a memory layout
   4458 		 * and map address which does not suit the main program.
   4459 		 */
   4460 		rc = mdb_env_init_meta(env, &meta);
   4461 		if (rc)
   4462 			return rc;
   4463 		newenv = 0;
   4464 	}
   4465 
   4466 	rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL);
   4467 	if (rc)
   4468 		return rc;
   4469 
   4470 	if (newenv) {
   4471 		if (flags & MDB_FIXEDMAP)
   4472 			meta.mm_address = env->me_map;
   4473 		i = mdb_env_init_meta(env, &meta);
   4474 		if (i != MDB_SUCCESS) {
   4475 			return i;
   4476 		}
   4477 	}
   4478 
   4479 	env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1;
   4480 	env->me_nodemax = (((env->me_psize - PAGEHDRSZ) / MDB_MINKEYS) & -2)
   4481 		- sizeof(indx_t);
   4482 #if !(MDB_MAXKEYSIZE)
   4483 	env->me_maxkey = env->me_nodemax - (NODESIZE + sizeof(MDB_db));
   4484 #endif
   4485 	env->me_maxpg = env->me_mapsize / env->me_psize;
   4486 
   4487 #if MDB_DEBUG
   4488 	{
   4489 		MDB_meta *meta = mdb_env_pick_meta(env);
   4490 		MDB_db *db = &meta->mm_dbs[MAIN_DBI];
   4491 
   4492 		DPRINTF(("opened database version %u, pagesize %u",
   4493 			meta->mm_version, env->me_psize));
   4494 		DPRINTF(("using meta page %d",    (int) (meta->mm_txnid & 1)));
   4495 		DPRINTF(("depth: %u",             db->md_depth));
   4496 		DPRINTF(("entries: %"Z"u",        db->md_entries));
   4497 		DPRINTF(("branch pages: %"Z"u",   db->md_branch_pages));
   4498 		DPRINTF(("leaf pages: %"Z"u",     db->md_leaf_pages));
   4499 		DPRINTF(("overflow pages: %"Z"u", db->md_overflow_pages));
   4500 		DPRINTF(("root: %"Z"u",           db->md_root));
   4501 	}
   4502 #endif
   4503 
   4504 	return MDB_SUCCESS;
   4505 }
   4506 
   4507 
   4508 /** Release a reader thread's slot in the reader lock table.
   4509  *	This function is called automatically when a thread exits.
   4510  * @param[in] ptr This points to the slot in the reader lock table.
   4511  */
   4512 static void
   4513 mdb_env_reader_dest(void *ptr)
   4514 {
   4515 	MDB_reader *reader = ptr;
   4516 
   4517 #ifndef _WIN32
   4518 	if (reader->mr_pid == getpid()) /* catch pthread_exit() in child process */
   4519 #endif
   4520 		/* We omit the mutex, so do this atomically (i.e. skip mr_txnid) */
   4521 		reader->mr_pid = 0;
   4522 }
   4523 
   4524 #ifdef _WIN32
   4525 /** Junk for arranging thread-specific callbacks on Windows. This is
   4526  *	necessarily platform and compiler-specific. Windows supports up
   4527  *	to 1088 keys. Let's assume nobody opens more than 64 environments
   4528  *	in a single process, for now. They can override this if needed.
   4529  */
   4530 #ifndef MAX_TLS_KEYS
   4531 #define MAX_TLS_KEYS	64
   4532 #endif
   4533 static pthread_key_t mdb_tls_keys[MAX_TLS_KEYS];
   4534 static int mdb_tls_nkeys;
   4535 
   4536 static void NTAPI mdb_tls_callback(PVOID module, DWORD reason, PVOID ptr)
   4537 {
   4538 	int i;
   4539 	switch(reason) {
   4540 	case DLL_PROCESS_ATTACH: break;
   4541 	case DLL_THREAD_ATTACH: break;
   4542 	case DLL_THREAD_DETACH:
   4543 		for (i=0; i<mdb_tls_nkeys; i++) {
   4544 			MDB_reader *r = pthread_getspecific(mdb_tls_keys[i]);
   4545 			if (r) {
   4546 				mdb_env_reader_dest(r);
   4547 			}
   4548 		}
   4549 		break;
   4550 	case DLL_PROCESS_DETACH: break;
   4551 	}
   4552 }
   4553 #ifdef __GNUC__
   4554 #ifdef _WIN64
   4555 const PIMAGE_TLS_CALLBACK mdb_tls_cbp __attribute__((section (".CRT$XLB"))) = mdb_tls_callback;
   4556 #else
   4557 PIMAGE_TLS_CALLBACK mdb_tls_cbp __attribute__((section (".CRT$XLB"))) = mdb_tls_callback;
   4558 #endif
   4559 #else
   4560 #ifdef _WIN64
   4561 /* Force some symbol references.
   4562  *	_tls_used forces the linker to create the TLS directory if not already done
   4563  *	mdb_tls_cbp prevents whole-program-optimizer from dropping the symbol.
   4564  */
   4565 #pragma comment(linker, "/INCLUDE:_tls_used")
   4566 #pragma comment(linker, "/INCLUDE:mdb_tls_cbp")
   4567 #pragma const_seg(".CRT$XLB")
   4568 extern const PIMAGE_TLS_CALLBACK mdb_tls_cbp;
   4569 const PIMAGE_TLS_CALLBACK mdb_tls_cbp = mdb_tls_callback;
   4570 #pragma const_seg()
   4571 #else	/* _WIN32 */
   4572 #pragma comment(linker, "/INCLUDE:__tls_used")
   4573 #pragma comment(linker, "/INCLUDE:_mdb_tls_cbp")
   4574 #pragma data_seg(".CRT$XLB")
   4575 PIMAGE_TLS_CALLBACK mdb_tls_cbp = mdb_tls_callback;
   4576 #pragma data_seg()
   4577 #endif	/* WIN 32/64 */
   4578 #endif	/* !__GNUC__ */
   4579 #endif
   4580 
   4581 /** Downgrade the exclusive lock on the region back to shared */
   4582 static int ESECT
   4583 mdb_env_share_locks(MDB_env *env, int *excl)
   4584 {
   4585 	int rc = 0;
   4586 	MDB_meta *meta = mdb_env_pick_meta(env);
   4587 
   4588 	env->me_txns->mti_txnid = meta->mm_txnid;
   4589 
   4590 #ifdef _WIN32
   4591 	{
   4592 		OVERLAPPED ov;
   4593 		/* First acquire a shared lock. The Unlock will
   4594 		 * then release the existing exclusive lock.
   4595 		 */
   4596 		memset(&ov, 0, sizeof(ov));
   4597 		if (!LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) {
   4598 			rc = ErrCode();
   4599 		} else {
   4600 			UnlockFile(env->me_lfd, 0, 0, 1, 0);
   4601 			*excl = 0;
   4602 		}
   4603 	}
   4604 #else
   4605 	{
   4606 		struct flock lock_info;
   4607 		/* The shared lock replaces the existing lock */
   4608 		memset((void *)&lock_info, 0, sizeof(lock_info));
   4609 		lock_info.l_type = F_RDLCK;
   4610 		lock_info.l_whence = SEEK_SET;
   4611 		lock_info.l_start = 0;
   4612 		lock_info.l_len = 1;
   4613 		while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) &&
   4614 				(rc = ErrCode()) == EINTR) ;
   4615 		*excl = rc ? -1 : 0;	/* error may mean we lost the lock */
   4616 	}
   4617 #endif
   4618 
   4619 	return rc;
   4620 }
   4621 
   4622 /** Try to get exclusive lock, otherwise shared.
   4623  *	Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive.
   4624  */
   4625 static int ESECT
   4626 mdb_env_excl_lock(MDB_env *env, int *excl)
   4627 {
   4628 	int rc = 0;
   4629 #ifdef _WIN32
   4630 	if (LockFile(env->me_lfd, 0, 0, 1, 0)) {
   4631 		*excl = 1;
   4632 	} else {
   4633 		OVERLAPPED ov;
   4634 		memset(&ov, 0, sizeof(ov));
   4635 		if (LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) {
   4636 			*excl = 0;
   4637 		} else {
   4638 			rc = ErrCode();
   4639 		}
   4640 	}
   4641 #else
   4642 	struct flock lock_info;
   4643 	memset((void *)&lock_info, 0, sizeof(lock_info));
   4644 	lock_info.l_type = F_WRLCK;
   4645 	lock_info.l_whence = SEEK_SET;
   4646 	lock_info.l_start = 0;
   4647 	lock_info.l_len = 1;
   4648 	while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) &&
   4649 			(rc = ErrCode()) == EINTR) ;
   4650 	if (!rc) {
   4651 		*excl = 1;
   4652 	} else
   4653 # ifndef MDB_USE_POSIX_MUTEX
   4654 	if (*excl < 0) /* always true when MDB_USE_POSIX_MUTEX */
   4655 # endif
   4656 	{
   4657 		lock_info.l_type = F_RDLCK;
   4658 		while ((rc = fcntl(env->me_lfd, F_SETLKW, &lock_info)) &&
   4659 				(rc = ErrCode()) == EINTR) ;
   4660 		if (rc == 0)
   4661 			*excl = 0;
   4662 	}
   4663 #endif
   4664 	return rc;
   4665 }
   4666 
   4667 #ifdef MDB_USE_HASH
   4668 /*
   4669  * hash_64 - 64 bit Fowler/Noll/Vo-0 FNV-1a hash code
   4670  *
   4671  * @(#) $Revision: 5.1 $
   4672  * @(#) $Id: hash_64a.c,v 5.1 2009/06/30 09:01:38 chongo Exp $
   4673  * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_64a.c,v $
   4674  *
   4675  *	  http://www.isthe.com/chongo/tech/comp/fnv/index.html
   4676  *
   4677  ***
   4678  *
   4679  * Please do not copyright this code.  This code is in the public domain.
   4680  *
   4681  * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
   4682  * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO
   4683  * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR
   4684  * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
   4685  * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
   4686  * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
   4687  * PERFORMANCE OF THIS SOFTWARE.
   4688  *
   4689  * By:
   4690  *	chongo <Landon Curt Noll> /\oo/\
   4691  *	  http://www.isthe.com/chongo/
   4692  *
   4693  * Share and Enjoy!	:-)
   4694  */
   4695 
   4696 typedef unsigned long long	mdb_hash_t;
   4697 #define MDB_HASH_INIT ((mdb_hash_t)0xcbf29ce484222325ULL)
   4698 
   4699 /** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer
   4700  * @param[in] val	value to hash
   4701  * @param[in] hval	initial value for hash
   4702  * @return 64 bit hash
   4703  *
   4704  * NOTE: To use the recommended 64 bit FNV-1a hash, use MDB_HASH_INIT as the
   4705  * 	 hval arg on the first call.
   4706  */
   4707 static mdb_hash_t
   4708 mdb_hash_val(MDB_val *val, mdb_hash_t hval)
   4709 {
   4710 	unsigned char *s = (unsigned char *)val->mv_data;	/* unsigned string */
   4711 	unsigned char *end = s + val->mv_size;
   4712 	/*
   4713 	 * FNV-1a hash each octet of the string
   4714 	 */
   4715 	while (s < end) {
   4716 		/* xor the bottom with the current octet */
   4717 		hval ^= (mdb_hash_t)*s++;
   4718 
   4719 		/* multiply by the 64 bit FNV magic prime mod 2^64 */
   4720 		hval += (hval << 1) + (hval << 4) + (hval << 5) +
   4721 			(hval << 7) + (hval << 8) + (hval << 40);
   4722 	}
   4723 	/* return our new hash value */
   4724 	return hval;
   4725 }
   4726 
   4727 /** Hash the string and output the encoded hash.
   4728  * This uses modified RFC1924 Ascii85 encoding to accommodate systems with
   4729  * very short name limits. We don't care about the encoding being reversible,
   4730  * we just want to preserve as many bits of the input as possible in a
   4731  * small printable string.
   4732  * @param[in] str string to hash
   4733  * @param[out] encbuf an array of 11 chars to hold the hash
   4734  */
   4735 static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~";
   4736 
   4737 static void ESECT
   4738 mdb_pack85(unsigned long l, char *out)
   4739 {
   4740 	int i;
   4741 
   4742 	for (i=0; i<5; i++) {
   4743 		*out++ = mdb_a85[l % 85];
   4744 		l /= 85;
   4745 	}
   4746 }
   4747 
   4748 static void ESECT
   4749 mdb_hash_enc(MDB_val *val, char *encbuf)
   4750 {
   4751 	mdb_hash_t h = mdb_hash_val(val, MDB_HASH_INIT);
   4752 
   4753 	mdb_pack85(h, encbuf);
   4754 	mdb_pack85(h>>32, encbuf+5);
   4755 	encbuf[10] = '\0';
   4756 }
   4757 #endif
   4758 
   4759 /** Open and/or initialize the lock region for the environment.
   4760  * @param[in] env The LMDB environment.
   4761  * @param[in] fname Filename + scratch area, from #mdb_fname_init().
   4762  * @param[in] mode The Unix permissions for the file, if we create it.
   4763  * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive
   4764  * @return 0 on success, non-zero on failure.
   4765  */
   4766 static int ESECT
   4767 mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
   4768 {
   4769 #ifdef _WIN32
   4770 #	define MDB_ERRCODE_ROFS	ERROR_WRITE_PROTECT
   4771 #else
   4772 #	define MDB_ERRCODE_ROFS	EROFS
   4773 #endif
   4774 	int rc;
   4775 	off_t size, rsize;
   4776 
   4777 	rc = mdb_fopen(env, fname, MDB_O_LOCKS, mode, &env->me_lfd);
   4778 	if (rc) {
   4779 		/* Omit lockfile if read-only env on read-only filesystem */
   4780 		if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) {
   4781 			return MDB_SUCCESS;
   4782 		}
   4783 		goto fail;
   4784 	}
   4785 
   4786 	if (!(env->me_flags & MDB_NOTLS)) {
   4787 		rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest);
   4788 		if (rc)
   4789 			goto fail;
   4790 		env->me_flags |= MDB_ENV_TXKEY;
   4791 #ifdef _WIN32
   4792 		/* Windows TLS callbacks need help finding their TLS info. */
   4793 		if (mdb_tls_nkeys >= MAX_TLS_KEYS) {
   4794 			rc = MDB_TLS_FULL;
   4795 			goto fail;
   4796 		}
   4797 		mdb_tls_keys[mdb_tls_nkeys++] = env->me_txkey;
   4798 #endif
   4799 	}
   4800 
   4801 	/* Try to get exclusive lock. If we succeed, then
   4802 	 * nobody is using the lock region and we should initialize it.
   4803 	 */
   4804 	if ((rc = mdb_env_excl_lock(env, excl))) goto fail;
   4805 
   4806 #ifdef _WIN32
   4807 	size = GetFileSize(env->me_lfd, NULL);
   4808 #else
   4809 	size = lseek(env->me_lfd, 0, SEEK_END);
   4810 	if (size == -1) goto fail_errno;
   4811 #endif
   4812 	rsize = (env->me_maxreaders-1) * sizeof(MDB_reader) + sizeof(MDB_txninfo);
   4813 	if (size < rsize && *excl > 0) {
   4814 #ifdef _WIN32
   4815 		if (SetFilePointer(env->me_lfd, rsize, NULL, FILE_BEGIN) != (DWORD)rsize
   4816 			|| !SetEndOfFile(env->me_lfd))
   4817 			goto fail_errno;
   4818 #else
   4819 		if (ftruncate(env->me_lfd, rsize) != 0) goto fail_errno;
   4820 #endif
   4821 	} else {
   4822 		rsize = size;
   4823 		size = rsize - sizeof(MDB_txninfo);
   4824 		env->me_maxreaders = size/sizeof(MDB_reader) + 1;
   4825 	}
   4826 	{
   4827 #ifdef _WIN32
   4828 		HANDLE mh;
   4829 		mh = CreateFileMapping(env->me_lfd, NULL, PAGE_READWRITE,
   4830 			0, 0, NULL);
   4831 		if (!mh) goto fail_errno;
   4832 		env->me_txns = MapViewOfFileEx(mh, FILE_MAP_WRITE, 0, 0, rsize, NULL);
   4833 		CloseHandle(mh);
   4834 		if (!env->me_txns) goto fail_errno;
   4835 #else
   4836 		void *m = mmap(NULL, rsize, PROT_READ|PROT_WRITE, MAP_SHARED,
   4837 			env->me_lfd, 0);
   4838 		if (m == MAP_FAILED) goto fail_errno;
   4839 		env->me_txns = m;
   4840 #endif
   4841 	}
   4842 	if (*excl > 0) {
   4843 #ifdef _WIN32
   4844 		BY_HANDLE_FILE_INFORMATION stbuf;
   4845 		struct {
   4846 			DWORD volume;
   4847 			DWORD nhigh;
   4848 			DWORD nlow;
   4849 		} idbuf;
   4850 		MDB_val val;
   4851 		char encbuf[11];
   4852 
   4853 		if (!mdb_sec_inited) {
   4854 			InitializeSecurityDescriptor(&mdb_null_sd,
   4855 				SECURITY_DESCRIPTOR_REVISION);
   4856 			SetSecurityDescriptorDacl(&mdb_null_sd, TRUE, 0, FALSE);
   4857 			mdb_all_sa.nLength = sizeof(SECURITY_ATTRIBUTES);
   4858 			mdb_all_sa.bInheritHandle = FALSE;
   4859 			mdb_all_sa.lpSecurityDescriptor = &mdb_null_sd;
   4860 			mdb_sec_inited = 1;
   4861 		}
   4862 		if (!GetFileInformationByHandle(env->me_lfd, &stbuf)) goto fail_errno;
   4863 		idbuf.volume = stbuf.dwVolumeSerialNumber;
   4864 		idbuf.nhigh  = stbuf.nFileIndexHigh;
   4865 		idbuf.nlow   = stbuf.nFileIndexLow;
   4866 		val.mv_data = &idbuf;
   4867 		val.mv_size = sizeof(idbuf);
   4868 		mdb_hash_enc(&val, encbuf);
   4869 		sprintf(env->me_txns->mti_rmname, "Global\\MDBr%s", encbuf);
   4870 		sprintf(env->me_txns->mti_wmname, "Global\\MDBw%s", encbuf);
   4871 		env->me_rmutex = CreateMutexA(&mdb_all_sa, FALSE, env->me_txns->mti_rmname);
   4872 		if (!env->me_rmutex) goto fail_errno;
   4873 		env->me_wmutex = CreateMutexA(&mdb_all_sa, FALSE, env->me_txns->mti_wmname);
   4874 		if (!env->me_wmutex) goto fail_errno;
   4875 #elif defined(MDB_USE_POSIX_SEM)
   4876 		struct stat stbuf;
   4877 		struct {
   4878 			dev_t dev;
   4879 			ino_t ino;
   4880 		} idbuf;
   4881 		MDB_val val;
   4882 		char encbuf[11];
   4883 
   4884 #if defined(__NetBSD__)
   4885 #define	MDB_SHORT_SEMNAMES	1	/* limited to 14 chars */
   4886 #endif
   4887 		if (fstat(env->me_lfd, &stbuf)) goto fail_errno;
   4888 		idbuf.dev = stbuf.st_dev;
   4889 		idbuf.ino = stbuf.st_ino;
   4890 		val.mv_data = &idbuf;
   4891 		val.mv_size = sizeof(idbuf);
   4892 		mdb_hash_enc(&val, encbuf);
   4893 #ifdef MDB_SHORT_SEMNAMES
   4894 		encbuf[9] = '\0';	/* drop name from 15 chars to 14 chars */
   4895 #endif
   4896 		sprintf(env->me_txns->mti_rmname, "/MDBr%s", encbuf);
   4897 		sprintf(env->me_txns->mti_wmname, "/MDBw%s", encbuf);
   4898 		/* Clean up after a previous run, if needed:  Try to
   4899 		 * remove both semaphores before doing anything else.
   4900 		 */
   4901 		sem_unlink(env->me_txns->mti_rmname);
   4902 		sem_unlink(env->me_txns->mti_wmname);
   4903 		env->me_rmutex = sem_open(env->me_txns->mti_rmname,
   4904 			O_CREAT|O_EXCL, mode, 1);
   4905 		if (env->me_rmutex == SEM_FAILED) goto fail_errno;
   4906 		env->me_wmutex = sem_open(env->me_txns->mti_wmname,
   4907 			O_CREAT|O_EXCL, mode, 1);
   4908 		if (env->me_wmutex == SEM_FAILED) goto fail_errno;
   4909 #else	/* MDB_USE_POSIX_MUTEX: */
   4910 		pthread_mutexattr_t mattr;
   4911 
   4912 		/* Solaris needs this before initing a robust mutex.  Otherwise
   4913 		 * it may skip the init and return EBUSY "seems someone already
   4914 		 * inited" or EINVAL "it was inited differently".
   4915 		 */
   4916 		memset(env->me_txns->mti_rmutex, 0, sizeof(*env->me_txns->mti_rmutex));
   4917 		memset(env->me_txns->mti_wmutex, 0, sizeof(*env->me_txns->mti_wmutex));
   4918 
   4919 		if ((rc = pthread_mutexattr_init(&mattr)))
   4920 			goto fail;
   4921 
   4922 		rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
   4923 #ifdef MDB_ROBUST_SUPPORTED
   4924 		if (!rc) rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST);
   4925 #endif
   4926 		if (!rc) rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr);
   4927 		if (!rc) rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr);
   4928 		pthread_mutexattr_destroy(&mattr);
   4929 		if (rc)
   4930 			goto fail;
   4931 #endif	/* _WIN32 || MDB_USE_POSIX_SEM */
   4932 
   4933 		env->me_txns->mti_magic = MDB_MAGIC;
   4934 		env->me_txns->mti_format = MDB_LOCK_FORMAT;
   4935 		env->me_txns->mti_txnid = 0;
   4936 		env->me_txns->mti_numreaders = 0;
   4937 
   4938 	} else {
   4939 		if (env->me_txns->mti_magic != MDB_MAGIC) {
   4940 			DPUTS("lock region has invalid magic");
   4941 			rc = MDB_INVALID;
   4942 			goto fail;
   4943 		}
   4944 		if (env->me_txns->mti_format != MDB_LOCK_FORMAT) {
   4945 			DPRINTF(("lock region has format+version 0x%x, expected 0x%x",
   4946 				env->me_txns->mti_format, MDB_LOCK_FORMAT));
   4947 			rc = MDB_VERSION_MISMATCH;
   4948 			goto fail;
   4949 		}
   4950 		rc = ErrCode();
   4951 		if (rc && rc != EACCES && rc != EAGAIN) {
   4952 			goto fail;
   4953 		}
   4954 #ifdef _WIN32
   4955 		env->me_rmutex = OpenMutexA(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname);
   4956 		if (!env->me_rmutex) goto fail_errno;
   4957 		env->me_wmutex = OpenMutexA(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname);
   4958 		if (!env->me_wmutex) goto fail_errno;
   4959 #elif defined(MDB_USE_POSIX_SEM)
   4960 		env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0);
   4961 		if (env->me_rmutex == SEM_FAILED) goto fail_errno;
   4962 		env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0);
   4963 		if (env->me_wmutex == SEM_FAILED) goto fail_errno;
   4964 #endif
   4965 	}
   4966 	return MDB_SUCCESS;
   4967 
   4968 fail_errno:
   4969 	rc = ErrCode();
   4970 fail:
   4971 	return rc;
   4972 }
   4973 
   4974 	/** Only a subset of the @ref mdb_env flags can be changed
   4975 	 *	at runtime. Changing other flags requires closing the
   4976 	 *	environment and re-opening it with the new flags.
   4977 	 */
   4978 #define	CHANGEABLE	(MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT)
   4979 #define	CHANGELESS	(MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \
   4980 	MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD)
   4981 
   4982 #if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS)
   4983 # error "Persistent DB flags & env flags overlap, but both go in mm_flags"
   4984 #endif
   4985 
   4986 int ESECT
   4987 mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode)
   4988 {
   4989 	int rc, excl = -1;
   4990 	MDB_name fname;
   4991 
   4992 	if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS)))
   4993 		return EINVAL;
   4994 
   4995 	flags |= env->me_flags;
   4996 
   4997 	rc = mdb_fname_init(path, flags, &fname);
   4998 	if (rc)
   4999 		return rc;
   5000 
   5001 	if (flags & MDB_RDONLY) {
   5002 		/* silently ignore WRITEMAP when we're only getting read access */
   5003 		flags &= ~MDB_WRITEMAP;
   5004 	} else {
   5005 		if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) &&
   5006 			  (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2)))))
   5007 			rc = ENOMEM;
   5008 	}
   5009 	env->me_flags = flags |= MDB_ENV_ACTIVE;
   5010 	if (rc)
   5011 		goto leave;
   5012 
   5013 	env->me_path = strdup(path);
   5014 	env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx));
   5015 	env->me_dbflags = calloc(env->me_maxdbs, sizeof(uint16_t));
   5016 	env->me_dbiseqs = calloc(env->me_maxdbs, sizeof(unsigned int));
   5017 	if (!(env->me_dbxs && env->me_path && env->me_dbflags && env->me_dbiseqs)) {
   5018 		rc = ENOMEM;
   5019 		goto leave;
   5020 	}
   5021 	env->me_dbxs[FREE_DBI].md_cmp = mdb_cmp_long; /* aligned MDB_INTEGERKEY */
   5022 
   5023 	/* For RDONLY, get lockfile after we know datafile exists */
   5024 	if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) {
   5025 		rc = mdb_env_setup_locks(env, &fname, mode, &excl);
   5026 		if (rc)
   5027 			goto leave;
   5028 	}
   5029 
   5030 	rc = mdb_fopen(env, &fname,
   5031 		(flags & MDB_RDONLY) ? MDB_O_RDONLY : MDB_O_RDWR,
   5032 		mode, &env->me_fd);
   5033 	if (rc)
   5034 		goto leave;
   5035 
   5036 	if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) {
   5037 		rc = mdb_env_setup_locks(env, &fname, mode, &excl);
   5038 		if (rc)
   5039 			goto leave;
   5040 	}
   5041 
   5042 	if ((rc = mdb_env_open2(env)) == MDB_SUCCESS) {
   5043 		if (!(flags & (MDB_RDONLY|MDB_WRITEMAP))) {
   5044 			/* Synchronous fd for meta writes. Needed even with
   5045 			 * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset.
   5046 			 */
   5047 			rc = mdb_fopen(env, &fname, MDB_O_META, mode, &env->me_mfd);
   5048 			if (rc)
   5049 				goto leave;
   5050 		}
   5051 		DPRINTF(("opened dbenv %p", (void *) env));
   5052 		if (excl > 0) {
   5053 			rc = mdb_env_share_locks(env, &excl);
   5054 			if (rc)
   5055 				goto leave;
   5056 		}
   5057 		if (!(flags & MDB_RDONLY)) {
   5058 			MDB_txn *txn;
   5059 			int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs *
   5060 				(sizeof(MDB_db)+sizeof(MDB_cursor *)+sizeof(unsigned int)+1);
   5061 			if ((env->me_pbuf = calloc(1, env->me_psize)) &&
   5062 				(txn = calloc(1, size)))
   5063 			{
   5064 				txn->mt_dbs = (MDB_db *)((char *)txn + tsize);
   5065 				txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs);
   5066 				txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs);
   5067 				txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs);
   5068 				txn->mt_env = env;
   5069 				txn->mt_dbxs = env->me_dbxs;
   5070 				txn->mt_flags = MDB_TXN_FINISHED;
   5071 				env->me_txn0 = txn;
   5072 			} else {
   5073 				rc = ENOMEM;
   5074 			}
   5075 		}
   5076 	}
   5077 
   5078 leave:
   5079 	if (rc) {
   5080 		mdb_env_close0(env, excl);
   5081 	}
   5082 	mdb_fname_destroy(fname);
   5083 	return rc;
   5084 }
   5085 
   5086 /** Destroy resources from mdb_env_open(), clear our readers & DBIs */
   5087 static void ESECT
   5088 mdb_env_close0(MDB_env *env, int excl)
   5089 {
   5090 	int i;
   5091 
   5092 	if (!(env->me_flags & MDB_ENV_ACTIVE))
   5093 		return;
   5094 
   5095 	/* Doing this here since me_dbxs may not exist during mdb_env_close */
   5096 	if (env->me_dbxs) {
   5097 		for (i = env->me_maxdbs; --i >= CORE_DBS; )
   5098 			free(env->me_dbxs[i].md_name.mv_data);
   5099 		free(env->me_dbxs);
   5100 	}
   5101 
   5102 	free(env->me_pbuf);
   5103 	free(env->me_dbiseqs);
   5104 	free(env->me_dbflags);
   5105 	free(env->me_path);
   5106 	free(env->me_dirty_list);
   5107 	free(env->me_txn0);
   5108 	mdb_midl_free(env->me_free_pgs);
   5109 
   5110 	if (env->me_flags & MDB_ENV_TXKEY) {
   5111 		pthread_key_delete(env->me_txkey);
   5112 #ifdef _WIN32
   5113 		/* Delete our key from the global list */
   5114 		for (i=0; i<mdb_tls_nkeys; i++)
   5115 			if (mdb_tls_keys[i] == env->me_txkey) {
   5116 				mdb_tls_keys[i] = mdb_tls_keys[mdb_tls_nkeys-1];
   5117 				mdb_tls_nkeys--;
   5118 				break;
   5119 			}
   5120 #endif
   5121 	}
   5122 
   5123 	if (env->me_map) {
   5124 		munmap(env->me_map, env->me_mapsize);
   5125 	}
   5126 	if (env->me_mfd != INVALID_HANDLE_VALUE)
   5127 		(void) close(env->me_mfd);
   5128 	if (env->me_fd != INVALID_HANDLE_VALUE)
   5129 		(void) close(env->me_fd);
   5130 	if (env->me_txns) {
   5131 		MDB_PID_T pid = getpid();
   5132 		/* Clearing readers is done in this function because
   5133 		 * me_txkey with its destructor must be disabled first.
   5134 		 *
   5135 		 * We skip the the reader mutex, so we touch only
   5136 		 * data owned by this process (me_close_readers and
   5137 		 * our readers), and clear each reader atomically.
   5138 		 */
   5139 		for (i = env->me_close_readers; --i >= 0; )
   5140 			if (env->me_txns->mti_readers[i].mr_pid == pid)
   5141 				env->me_txns->mti_readers[i].mr_pid = 0;
   5142 #ifdef _WIN32
   5143 		if (env->me_rmutex) {
   5144 			CloseHandle(env->me_rmutex);
   5145 			if (env->me_wmutex) CloseHandle(env->me_wmutex);
   5146 		}
   5147 		/* Windows automatically destroys the mutexes when
   5148 		 * the last handle closes.
   5149 		 */
   5150 #elif defined(MDB_USE_POSIX_SEM)
   5151 		if (env->me_rmutex != SEM_FAILED) {
   5152 			sem_close(env->me_rmutex);
   5153 			if (env->me_wmutex != SEM_FAILED)
   5154 				sem_close(env->me_wmutex);
   5155 			/* If we have the filelock:  If we are the
   5156 			 * only remaining user, clean up semaphores.
   5157 			 */
   5158 			if (excl == 0)
   5159 				mdb_env_excl_lock(env, &excl);
   5160 			if (excl > 0) {
   5161 				sem_unlink(env->me_txns->mti_rmname);
   5162 				sem_unlink(env->me_txns->mti_wmname);
   5163 			}
   5164 		}
   5165 #elif defined(MDB_ROBUST_SUPPORTED)
   5166 		/* If we have the filelock:  If we are the
   5167 		 * only remaining user, clean up robust
   5168 		 * mutexes.
   5169 		 */
   5170 		if (excl == 0)
   5171 			mdb_env_excl_lock(env, &excl);
   5172 		if (excl > 0) {
   5173 			pthread_mutex_destroy(env->me_txns->mti_rmutex);
   5174 			pthread_mutex_destroy(env->me_txns->mti_wmutex);
   5175 		}
   5176 #endif
   5177 		munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo));
   5178 	}
   5179 	if (env->me_lfd != INVALID_HANDLE_VALUE) {
   5180 #ifdef _WIN32
   5181 		if (excl >= 0) {
   5182 			/* Unlock the lockfile.  Windows would have unlocked it
   5183 			 * after closing anyway, but not necessarily at once.
   5184 			 */
   5185 			UnlockFile(env->me_lfd, 0, 0, 1, 0);
   5186 		}
   5187 #endif
   5188 		(void) close(env->me_lfd);
   5189 	}
   5190 
   5191 	env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY);
   5192 }
   5193 
   5194 void ESECT
   5195 mdb_env_close(MDB_env *env)
   5196 {
   5197 	MDB_page *dp;
   5198 
   5199 	if (env == NULL)
   5200 		return;
   5201 
   5202 	VGMEMP_DESTROY(env);
   5203 	while ((dp = env->me_dpages) != NULL) {
   5204 		VGMEMP_DEFINED(&dp->mp_next, sizeof(dp->mp_next));
   5205 		env->me_dpages = dp->mp_next;
   5206 		free(dp);
   5207 	}
   5208 
   5209 	mdb_env_close0(env, 0);
   5210 	free(env);
   5211 }
   5212 
   5213 /** Compare two items pointing at aligned size_t's */
   5214 static int
   5215 mdb_cmp_long(const MDB_val *a, const MDB_val *b)
   5216 {
   5217 	return (*(size_t *)a->mv_data < *(size_t *)b->mv_data) ? -1 :
   5218 		*(size_t *)a->mv_data > *(size_t *)b->mv_data;
   5219 }
   5220 
   5221 /** Compare two items pointing at aligned unsigned int's.
   5222  *
   5223  *	This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp,
   5224  *	but #mdb_cmp_clong() is called instead if the data type is size_t.
   5225  */
   5226 static int
   5227 mdb_cmp_int(const MDB_val *a, const MDB_val *b)
   5228 {
   5229 	return (*(unsigned int *)a->mv_data < *(unsigned int *)b->mv_data) ? -1 :
   5230 		*(unsigned int *)a->mv_data > *(unsigned int *)b->mv_data;
   5231 }
   5232 
   5233 /** Compare two items pointing at unsigned ints of unknown alignment.
   5234  *	Nodes and keys are guaranteed to be 2-byte aligned.
   5235  */
   5236 static int
   5237 mdb_cmp_cint(const MDB_val *a, const MDB_val *b)
   5238 {
   5239 #if BYTE_ORDER == LITTLE_ENDIAN
   5240 	unsigned short *u, *c;
   5241 	int x;
   5242 
   5243 	u = (unsigned short *) ((char *) a->mv_data + a->mv_size);
   5244 	c = (unsigned short *) ((char *) b->mv_data + a->mv_size);
   5245 	do {
   5246 		x = *--u - *--c;
   5247 	} while(!x && u > (unsigned short *)a->mv_data);
   5248 	return x;
   5249 #else
   5250 	unsigned short *u, *c, *end;
   5251 	int x;
   5252 
   5253 	end = (unsigned short *) ((char *) a->mv_data + a->mv_size);
   5254 	u = (unsigned short *)a->mv_data;
   5255 	c = (unsigned short *)b->mv_data;
   5256 	do {
   5257 		x = *u++ - *c++;
   5258 	} while(!x && u < end);
   5259 	return x;
   5260 #endif
   5261 }
   5262 
   5263 /** Compare two items lexically */
   5264 static int
   5265 mdb_cmp_memn(const MDB_val *a, const MDB_val *b)
   5266 {
   5267 	int diff;
   5268 	ssize_t len_diff;
   5269 	unsigned int len;
   5270 
   5271 	len = a->mv_size;
   5272 	len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size;
   5273 	if (len_diff > 0) {
   5274 		len = b->mv_size;
   5275 		len_diff = 1;
   5276 	}
   5277 
   5278 	diff = memcmp(a->mv_data, b->mv_data, len);
   5279 	return diff ? diff : len_diff<0 ? -1 : len_diff;
   5280 }
   5281 
   5282 /** Compare two items in reverse byte order */
   5283 static int
   5284 mdb_cmp_memnr(const MDB_val *a, const MDB_val *b)
   5285 {
   5286 	const unsigned char	*p1, *p2, *p1_lim;
   5287 	ssize_t len_diff;
   5288 	int diff;
   5289 
   5290 	p1_lim = (const unsigned char *)a->mv_data;
   5291 	p1 = (const unsigned char *)a->mv_data + a->mv_size;
   5292 	p2 = (const unsigned char *)b->mv_data + b->mv_size;
   5293 
   5294 	len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size;
   5295 	if (len_diff > 0) {
   5296 		p1_lim += len_diff;
   5297 		len_diff = 1;
   5298 	}
   5299 
   5300 	while (p1 > p1_lim) {
   5301 		diff = *--p1 - *--p2;
   5302 		if (diff)
   5303 			return diff;
   5304 	}
   5305 	return len_diff<0 ? -1 : len_diff;
   5306 }
   5307 
   5308 /** Search for key within a page, using binary search.
   5309  * Returns the smallest entry larger or equal to the key.
   5310  * If exactp is non-null, stores whether the found entry was an exact match
   5311  * in *exactp (1 or 0).
   5312  * Updates the cursor index with the index of the found entry.
   5313  * If no entry larger or equal to the key is found, returns NULL.
   5314  */
   5315 static MDB_node *
   5316 mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp)
   5317 {
   5318 	unsigned int	 i = 0, nkeys;
   5319 	int		 low, high;
   5320 	int		 rc = 0;
   5321 	MDB_page *mp = mc->mc_pg[mc->mc_top];
   5322 	MDB_node	*node = NULL;
   5323 	MDB_val	 nodekey;
   5324 	MDB_cmp_func *cmp;
   5325 	DKBUF;
   5326 
   5327 	nkeys = NUMKEYS(mp);
   5328 
   5329 	DPRINTF(("searching %u keys in %s %spage %"Z"u",
   5330 	    nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "",
   5331 	    mdb_dbg_pgno(mp)));
   5332 
   5333 	low = IS_LEAF(mp) ? 0 : 1;
   5334 	high = nkeys - 1;
   5335 	cmp = mc->mc_dbx->md_cmp;
   5336 
   5337 	/* Branch pages have no data, so if using integer keys,
   5338 	 * alignment is guaranteed. Use faster mdb_cmp_int.
   5339 	 */
   5340 	if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) {
   5341 		if (NODEPTR(mp, 1)->mn_ksize == sizeof(size_t))
   5342 			cmp = mdb_cmp_long;
   5343 		else
   5344 			cmp = mdb_cmp_int;
   5345 	}
   5346 
   5347 	if (IS_LEAF2(mp)) {
   5348 		nodekey.mv_size = mc->mc_db->md_pad;
   5349 		node = NODEPTR(mp, 0);	/* fake */
   5350 		while (low <= high) {
   5351 			i = (low + high) >> 1;
   5352 			nodekey.mv_data = LEAF2KEY(mp, i, nodekey.mv_size);
   5353 			rc = cmp(key, &nodekey);
   5354 			DPRINTF(("found leaf index %u [%s], rc = %i",
   5355 			    i, DKEY(&nodekey), rc));
   5356 			if (rc == 0)
   5357 				break;
   5358 			if (rc > 0)
   5359 				low = i + 1;
   5360 			else
   5361 				high = i - 1;
   5362 		}
   5363 	} else {
   5364 		while (low <= high) {
   5365 			i = (low + high) >> 1;
   5366 
   5367 			node = NODEPTR(mp, i);
   5368 			nodekey.mv_size = NODEKSZ(node);
   5369 			nodekey.mv_data = NODEKEY(node);
   5370 
   5371 			rc = cmp(key, &nodekey);
   5372 #if MDB_DEBUG
   5373 			if (IS_LEAF(mp))
   5374 				DPRINTF(("found leaf index %u [%s], rc = %i",
   5375 				    i, DKEY(&nodekey), rc));
   5376 			else
   5377 				DPRINTF(("found branch index %u [%s -> %"Z"u], rc = %i",
   5378 				    i, DKEY(&nodekey), NODEPGNO(node), rc));
   5379 #endif
   5380 			if (rc == 0)
   5381 				break;
   5382 			if (rc > 0)
   5383 				low = i + 1;
   5384 			else
   5385 				high = i - 1;
   5386 		}
   5387 	}
   5388 
   5389 	if (rc > 0) {	/* Found entry is less than the key. */
   5390 		i++;	/* Skip to get the smallest entry larger than key. */
   5391 		if (!IS_LEAF2(mp))
   5392 			node = NODEPTR(mp, i);
   5393 	}
   5394 	if (exactp)
   5395 		*exactp = (rc == 0 && nkeys > 0);
   5396 	/* store the key index */
   5397 	mc->mc_ki[mc->mc_top] = i;
   5398 	if (i >= nkeys)
   5399 		/* There is no entry larger or equal to the key. */
   5400 		return NULL;
   5401 
   5402 	/* nodeptr is fake for LEAF2 */
   5403 	return node;
   5404 }
   5405 
   5406 #if 0
   5407 static void
   5408 mdb_cursor_adjust(MDB_cursor *mc, func)
   5409 {
   5410 	MDB_cursor *m2;
   5411 
   5412 	for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) {
   5413 		if (m2->mc_pg[m2->mc_top] == mc->mc_pg[mc->mc_top]) {
   5414 			func(mc, m2);
   5415 		}
   5416 	}
   5417 }
   5418 #endif
   5419 
   5420 /** Pop a page off the top of the cursor's stack. */
   5421 static void
   5422 mdb_cursor_pop(MDB_cursor *mc)
   5423 {
   5424 	if (mc->mc_snum) {
   5425 		DPRINTF(("popping page %"Z"u off db %d cursor %p",
   5426 			mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc));
   5427 
   5428 		mc->mc_snum--;
   5429 		if (mc->mc_snum) {
   5430 			mc->mc_top--;
   5431 		} else {
   5432 			mc->mc_flags &= ~C_INITIALIZED;
   5433 		}
   5434 	}
   5435 }
   5436 
   5437 /** Push a page onto the top of the cursor's stack.
   5438  * Set #MDB_TXN_ERROR on failure.
   5439  */
   5440 static int
   5441 mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
   5442 {
   5443 	DPRINTF(("pushing page %"Z"u on db %d cursor %p", mp->mp_pgno,
   5444 		DDBI(mc), (void *) mc));
   5445 
   5446 	if (mc->mc_snum >= CURSOR_STACK) {
   5447 		mc->mc_txn->mt_flags |= MDB_TXN_ERROR;
   5448 		return MDB_CURSOR_FULL;
   5449 	}
   5450 
   5451 	mc->mc_top = mc->mc_snum++;
   5452 	mc->mc_pg[mc->mc_top] = mp;
   5453 	mc->mc_ki[mc->mc_top] = 0;
   5454 
   5455 	return MDB_SUCCESS;
   5456 }
   5457 
   5458 /** Find the address of the page corresponding to a given page number.
   5459  * Set #MDB_TXN_ERROR on failure.
   5460  * @param[in] mc the cursor accessing the page.
   5461  * @param[in] pgno the page number for the page to retrieve.
   5462  * @param[out] ret address of a pointer where the page's address will be stored.
   5463  * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page.
   5464  * @return 0 on success, non-zero on failure.
   5465  */
   5466 static int
   5467 mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl)
   5468 {
   5469 	MDB_txn *txn = mc->mc_txn;
   5470 	MDB_env *env = txn->mt_env;
   5471 	MDB_page *p = NULL;
   5472 	int level;
   5473 
   5474 	if (! (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_WRITEMAP))) {
   5475 		MDB_txn *tx2 = txn;
   5476 		level = 1;
   5477 		do {
   5478 			MDB_ID2L dl = tx2->mt_u.dirty_list;
   5479 			unsigned x;
   5480 			/* Spilled pages were dirtied in this txn and flushed
   5481 			 * because the dirty list got full. Bring this page
   5482 			 * back in from the map (but don't unspill it here,
   5483 			 * leave that unless page_touch happens again).
   5484 			 */
   5485 			if (tx2->mt_spill_pgs) {
   5486 				MDB_ID pn = pgno << 1;
   5487 				x = mdb_midl_search(tx2->mt_spill_pgs, pn);
   5488 				if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) {
   5489 					p = (MDB_page *)(env->me_map + env->me_psize * pgno);
   5490 					goto done;
   5491 				}
   5492 			}
   5493 			if (dl[0].mid) {
   5494 				unsigned x = mdb_mid2l_search(dl, pgno);
   5495 				if (x <= dl[0].mid && dl[x].mid == pgno) {
   5496 					p = dl[x].mptr;
   5497 					goto done;
   5498 				}
   5499 			}
   5500 			level++;
   5501 		} while ((tx2 = tx2->mt_parent) != NULL);
   5502 	}
   5503 
   5504 	if (pgno < txn->mt_next_pgno) {
   5505 		level = 0;
   5506 		p = (MDB_page *)(env->me_map + env->me_psize * pgno);
   5507 	} else {
   5508 		DPRINTF(("page %"Z"u not found", pgno));
   5509 		txn->mt_flags |= MDB_TXN_ERROR;
   5510 		return MDB_PAGE_NOTFOUND;
   5511 	}
   5512 
   5513 done:
   5514 	*ret = p;
   5515 	if (lvl)
   5516 		*lvl = level;
   5517 	return MDB_SUCCESS;
   5518 }
   5519 
   5520 /** Finish #mdb_page_search() / #mdb_page_search_lowest().
   5521  *	The cursor is at the root page, set up the rest of it.
   5522  */
   5523 static int
   5524 mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
   5525 {
   5526 	MDB_page	*mp = mc->mc_pg[mc->mc_top];
   5527 	int rc;
   5528 	DKBUF;
   5529 
   5530 	while (IS_BRANCH(mp)) {
   5531 		MDB_node	*node;
   5532 		indx_t		i;
   5533 
   5534 		DPRINTF(("branch page %"Z"u has %u keys", mp->mp_pgno, NUMKEYS(mp)));
   5535 		/* Don't assert on branch pages in the FreeDB. We can get here
   5536 		 * while in the process of rebalancing a FreeDB branch page; we must
   5537 		 * let that proceed. ITS#8336
   5538 		 */
   5539 		mdb_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1);
   5540 		DPRINTF(("found index 0 to page %"Z"u", NODEPGNO(NODEPTR(mp, 0))));
   5541 
   5542 		if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) {
   5543 			i = 0;
   5544 			if (flags & MDB_PS_LAST) {
   5545 				i = NUMKEYS(mp) - 1;
   5546 				/* if already init'd, see if we're already in right place */
   5547 				if (mc->mc_flags & C_INITIALIZED) {
   5548 					if (mc->mc_ki[mc->mc_top] == i) {
   5549 						mc->mc_top = mc->mc_snum++;
   5550 						mp = mc->mc_pg[mc->mc_top];
   5551 						goto ready;
   5552 					}
   5553 				}
   5554 			}
   5555 		} else {
   5556 			int	 exact;
   5557 			node = mdb_node_search(mc, key, &exact);
   5558 			if (node == NULL)
   5559 				i = NUMKEYS(mp) - 1;
   5560 			else {
   5561 				i = mc->mc_ki[mc->mc_top];
   5562 				if (!exact) {
   5563 					mdb_cassert(mc, i > 0);
   5564 					i--;
   5565 				}
   5566 			}
   5567 			DPRINTF(("following index %u for key [%s]", i, DKEY(key)));
   5568 		}
   5569 
   5570 		mdb_cassert(mc, i < NUMKEYS(mp));
   5571 		node = NODEPTR(mp, i);
   5572 
   5573 		if ((rc = mdb_page_get(mc, NODEPGNO(node), &mp, NULL)) != 0)
   5574 			return rc;
   5575 
   5576 		mc->mc_ki[mc->mc_top] = i;
   5577 		if ((rc = mdb_cursor_push(mc, mp)))
   5578 			return rc;
   5579 
   5580 ready:
   5581 		if (flags & MDB_PS_MODIFY) {
   5582 			if ((rc = mdb_page_touch(mc)) != 0)
   5583 				return rc;
   5584 			mp = mc->mc_pg[mc->mc_top];
   5585 		}
   5586 	}
   5587 
   5588 	if (!IS_LEAF(mp)) {
   5589 		DPRINTF(("internal error, index points to a %02X page!?",
   5590 		    mp->mp_flags));
   5591 		mc->mc_txn->mt_flags |= MDB_TXN_ERROR;
   5592 		return MDB_CORRUPTED;
   5593 	}
   5594 
   5595 	DPRINTF(("found leaf page %"Z"u for key [%s]", mp->mp_pgno,
   5596 	    key ? DKEY(key) : "null"));
   5597 	mc->mc_flags |= C_INITIALIZED;
   5598 	mc->mc_flags &= ~C_EOF;
   5599 
   5600 	return MDB_SUCCESS;
   5601 }
   5602 
   5603 /** Search for the lowest key under the current branch page.
   5604  * This just bypasses a NUMKEYS check in the current page
   5605  * before calling mdb_page_search_root(), because the callers
   5606  * are all in situations where the current page is known to
   5607  * be underfilled.
   5608  */
   5609 static int
   5610 mdb_page_search_lowest(MDB_cursor *mc)
   5611 {
   5612 	MDB_page	*mp = mc->mc_pg[mc->mc_top];
   5613 	MDB_node	*node = NODEPTR(mp, 0);
   5614 	int rc;
   5615 
   5616 	if ((rc = mdb_page_get(mc, NODEPGNO(node), &mp, NULL)) != 0)
   5617 		return rc;
   5618 
   5619 	mc->mc_ki[mc->mc_top] = 0;
   5620 	if ((rc = mdb_cursor_push(mc, mp)))
   5621 		return rc;
   5622 	return mdb_page_search_root(mc, NULL, MDB_PS_FIRST);
   5623 }
   5624 
   5625 /** Search for the page a given key should be in.
   5626  * Push it and its parent pages on the cursor stack.
   5627  * @param[in,out] mc the cursor for this operation.
   5628  * @param[in] key the key to search for, or NULL for first/last page.
   5629  * @param[in] flags If MDB_PS_MODIFY is set, visited pages in the DB
   5630  *   are touched (updated with new page numbers).
   5631  *   If MDB_PS_FIRST or MDB_PS_LAST is set, find first or last leaf.
   5632  *   This is used by #mdb_cursor_first() and #mdb_cursor_last().
   5633  *   If MDB_PS_ROOTONLY set, just fetch root node, no further lookups.
   5634  * @return 0 on success, non-zero on failure.
   5635  */
   5636 static int
   5637 mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
   5638 {
   5639 	int		 rc;
   5640 	pgno_t		 root;
   5641 
   5642 	/* Make sure the txn is still viable, then find the root from
   5643 	 * the txn's db table and set it as the root of the cursor's stack.
   5644 	 */
   5645 	if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED) {
   5646 		DPUTS("transaction may not be used now");
   5647 		return MDB_BAD_TXN;
   5648 	} else {
   5649 		/* Make sure we're using an up-to-date root */
   5650 		if (*mc->mc_dbflag & DB_STALE) {
   5651 				MDB_cursor mc2;
   5652 				if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi))
   5653 					return MDB_BAD_DBI;
   5654 				mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL);
   5655 				rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, 0);
   5656 				if (rc)
   5657 					return rc;
   5658 				{
   5659 					MDB_val data;
   5660 					int exact = 0;
   5661 					uint16_t flags;
   5662 					MDB_node *leaf = mdb_node_search(&mc2,
   5663 						&mc->mc_dbx->md_name, &exact);
   5664 					if (!exact)
   5665 						return MDB_NOTFOUND;
   5666 					if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA)
   5667 						return MDB_INCOMPATIBLE; /* not a named DB */
   5668 					rc = mdb_node_read(&mc2, leaf, &data);
   5669 					if (rc)
   5670 						return rc;
   5671 					memcpy(&flags, ((char *) data.mv_data + offsetof(MDB_db, md_flags)),
   5672 						sizeof(uint16_t));
   5673 					/* The txn may not know this DBI, or another process may
   5674 					 * have dropped and recreated the DB with other flags.
   5675 					 */
   5676 					if ((mc->mc_db->md_flags & PERSISTENT_FLAGS) != flags)
   5677 						return MDB_INCOMPATIBLE;
   5678 					memcpy(mc->mc_db, data.mv_data, sizeof(MDB_db));
   5679 				}
   5680 				*mc->mc_dbflag &= ~DB_STALE;
   5681 		}
   5682 		root = mc->mc_db->md_root;
   5683 
   5684 		if (root == P_INVALID) {		/* Tree is empty. */
   5685 			DPUTS("tree is empty");
   5686 			return MDB_NOTFOUND;
   5687 		}
   5688 	}
   5689 
   5690 	mdb_cassert(mc, root > 1);
   5691 	if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root)
   5692 		if ((rc = mdb_page_get(mc, root, &mc->mc_pg[0], NULL)) != 0)
   5693 			return rc;
   5694 
   5695 	mc->mc_snum = 1;
   5696 	mc->mc_top = 0;
   5697 
   5698 	DPRINTF(("db %d root page %"Z"u has flags 0x%X",
   5699 		DDBI(mc), root, mc->mc_pg[0]->mp_flags));
   5700 
   5701 	if (flags & MDB_PS_MODIFY) {
   5702 		if ((rc = mdb_page_touch(mc)))
   5703 			return rc;
   5704 	}
   5705 
   5706 	if (flags & MDB_PS_ROOTONLY)
   5707 		return MDB_SUCCESS;
   5708 
   5709 	return mdb_page_search_root(mc, key, flags);
   5710 }
   5711 
   5712 static int
   5713 mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
   5714 {
   5715 	MDB_txn *txn = mc->mc_txn;
   5716 	pgno_t pg = mp->mp_pgno;
   5717 	unsigned x = 0, ovpages = mp->mp_pages;
   5718 	MDB_env *env = txn->mt_env;
   5719 	MDB_IDL sl = txn->mt_spill_pgs;
   5720 	MDB_ID pn = pg << 1;
   5721 	int rc;
   5722 
   5723 	DPRINTF(("free ov page %"Z"u (%d)", pg, ovpages));
   5724 	/* If the page is dirty or on the spill list we just acquired it,
   5725 	 * so we should give it back to our current free list, if any.
   5726 	 * Otherwise put it onto the list of pages we freed in this txn.
   5727 	 *
   5728 	 * Won't create me_pghead: me_pglast must be inited along with it.
   5729 	 * Unsupported in nested txns: They would need to hide the page
   5730 	 * range in ancestor txns' dirty and spilled lists.
   5731 	 */
   5732 	if (env->me_pghead &&
   5733 		!txn->mt_parent &&
   5734 		((mp->mp_flags & P_DIRTY) ||
   5735 		 (sl && (x = mdb_midl_search(sl, pn)) <= sl[0] && sl[x] == pn)))
   5736 	{
   5737 		unsigned i, j;
   5738 		pgno_t *mop;
   5739 		MDB_ID2 *dl, ix, iy;
   5740 		rc = mdb_midl_need(&env->me_pghead, ovpages);
   5741 		if (rc)
   5742 			return rc;
   5743 		if (!(mp->mp_flags & P_DIRTY)) {
   5744 			/* This page is no longer spilled */
   5745 			if (x == sl[0])
   5746 				sl[0]--;
   5747 			else
   5748 				sl[x] |= 1;
   5749 			goto release;
   5750 		}
   5751 		/* Remove from dirty list */
   5752 		dl = txn->mt_u.dirty_list;
   5753 		x = dl[0].mid--;
   5754 		for (ix = dl[x]; ix.mptr != mp; ix = iy) {
   5755 			if (x > 1) {
   5756 				x--;
   5757 				iy = dl[x];
   5758 				dl[x] = ix;
   5759 			} else {
   5760 				mdb_cassert(mc, x > 1);
   5761 				j = ++(dl[0].mid);
   5762 				dl[j] = ix;		/* Unsorted. OK when MDB_TXN_ERROR. */
   5763 				txn->mt_flags |= MDB_TXN_ERROR;
   5764 				return MDB_CORRUPTED;
   5765 			}
   5766 		}
   5767 		txn->mt_dirty_room++;
   5768 		if (!(env->me_flags & MDB_WRITEMAP))
   5769 			mdb_dpage_free(env, mp);
   5770 release:
   5771 		/* Insert in me_pghead */
   5772 		mop = env->me_pghead;
   5773 		j = mop[0] + ovpages;
   5774 		for (i = mop[0]; i && mop[i] < pg; i--)
   5775 			mop[j--] = mop[i];
   5776 		while (j>i)
   5777 			mop[j--] = pg++;
   5778 		mop[0] += ovpages;
   5779 	} else {
   5780 		rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages);
   5781 		if (rc)
   5782 			return rc;
   5783 	}
   5784 	mc->mc_db->md_overflow_pages -= ovpages;
   5785 	return 0;
   5786 }
   5787 
   5788 /** Return the data associated with a given node.
   5789  * @param[in] mc The cursor for this operation.
   5790  * @param[in] leaf The node being read.
   5791  * @param[out] data Updated to point to the node's data.
   5792  * @return 0 on success, non-zero on failure.
   5793  */
   5794 static int
   5795 mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
   5796 {
   5797 	MDB_page	*omp;		/* overflow page */
   5798 	pgno_t		 pgno;
   5799 	int rc;
   5800 
   5801 	if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) {
   5802 		data->mv_size = NODEDSZ(leaf);
   5803 		data->mv_data = NODEDATA(leaf);
   5804 		return MDB_SUCCESS;
   5805 	}
   5806 
   5807 	/* Read overflow data.
   5808 	 */
   5809 	data->mv_size = NODEDSZ(leaf);
   5810 	memcpy(&pgno, NODEDATA(leaf), sizeof(pgno));
   5811 	if ((rc = mdb_page_get(mc, pgno, &omp, NULL)) != 0) {
   5812 		DPRINTF(("read overflow page %"Z"u failed", pgno));
   5813 		return rc;
   5814 	}
   5815 	data->mv_data = METADATA(omp);
   5816 
   5817 	return MDB_SUCCESS;
   5818 }
   5819 
   5820 int
   5821 mdb_get(MDB_txn *txn, MDB_dbi dbi,
   5822     MDB_val *key, MDB_val *data)
   5823 {
   5824 	MDB_cursor	mc;
   5825 	MDB_xcursor	mx;
   5826 	int exact = 0;
   5827 	DKBUF;
   5828 
   5829 	DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key)));
   5830 
   5831 	if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
   5832 		return EINVAL;
   5833 
   5834 	if (txn->mt_flags & MDB_TXN_BLOCKED)
   5835 		return MDB_BAD_TXN;
   5836 
   5837 	mdb_cursor_init(&mc, txn, dbi, &mx);
   5838 	return mdb_cursor_set(&mc, key, data, MDB_SET, &exact);
   5839 }
   5840 
   5841 /** Find a sibling for a page.
   5842  * Replaces the page at the top of the cursor's stack with the
   5843  * specified sibling, if one exists.
   5844  * @param[in] mc The cursor for this operation.
   5845  * @param[in] move_right Non-zero if the right sibling is requested,
   5846  * otherwise the left sibling.
   5847  * @return 0 on success, non-zero on failure.
   5848  */
   5849 static int
   5850 mdb_cursor_sibling(MDB_cursor *mc, int move_right)
   5851 {
   5852 	int		 rc;
   5853 	MDB_node	*indx;
   5854 	MDB_page	*mp;
   5855 
   5856 	if (mc->mc_snum < 2) {
   5857 		return MDB_NOTFOUND;		/* root has no siblings */
   5858 	}
   5859 
   5860 	mdb_cursor_pop(mc);
   5861 	DPRINTF(("parent page is page %"Z"u, index %u",
   5862 		mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top]));
   5863 
   5864 	if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top]))
   5865 		       : (mc->mc_ki[mc->mc_top] == 0)) {
   5866 		DPRINTF(("no more keys left, moving to %s sibling",
   5867 		    move_right ? "right" : "left"));
   5868 		if ((rc = mdb_cursor_sibling(mc, move_right)) != MDB_SUCCESS) {
   5869 			/* undo cursor_pop before returning */
   5870 			mc->mc_top++;
   5871 			mc->mc_snum++;
   5872 			return rc;
   5873 		}
   5874 	} else {
   5875 		if (move_right)
   5876 			mc->mc_ki[mc->mc_top]++;
   5877 		else
   5878 			mc->mc_ki[mc->mc_top]--;
   5879 		DPRINTF(("just moving to %s index key %u",
   5880 		    move_right ? "right" : "left", mc->mc_ki[mc->mc_top]));
   5881 	}
   5882 	mdb_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top]));
   5883 
   5884 	indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
   5885 	if ((rc = mdb_page_get(mc, NODEPGNO(indx), &mp, NULL)) != 0) {
   5886 		/* mc will be inconsistent if caller does mc_snum++ as above */
   5887 		mc->mc_flags &= ~(C_INITIALIZED|C_EOF);
   5888 		return rc;
   5889 	}
   5890 
   5891 	mdb_cursor_push(mc, mp);
   5892 	if (!move_right)
   5893 		mc->mc_ki[mc->mc_top] = NUMKEYS(mp)-1;
   5894 
   5895 	return MDB_SUCCESS;
   5896 }
   5897 
   5898 /** Move the cursor to the next data item. */
   5899 static int
   5900 mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
   5901 {
   5902 	MDB_page	*mp;
   5903 	MDB_node	*leaf;
   5904 	int rc;
   5905 
   5906 	if ((mc->mc_flags & C_DEL && op == MDB_NEXT_DUP))
   5907 		return MDB_NOTFOUND;
   5908 
   5909 	if (!(mc->mc_flags & C_INITIALIZED))
   5910 		return mdb_cursor_first(mc, key, data);
   5911 
   5912 	mp = mc->mc_pg[mc->mc_top];
   5913 
   5914 	if (mc->mc_flags & C_EOF) {
   5915 		if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mp)-1)
   5916 			return MDB_NOTFOUND;
   5917 		mc->mc_flags ^= C_EOF;
   5918 	}
   5919 
   5920 	if (mc->mc_db->md_flags & MDB_DUPSORT) {
   5921 		leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
   5922 		if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   5923 			if (op == MDB_NEXT || op == MDB_NEXT_DUP) {
   5924 				rc = mdb_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_NEXT);
   5925 				if (op != MDB_NEXT || rc != MDB_NOTFOUND) {
   5926 					if (rc == MDB_SUCCESS)
   5927 						MDB_GET_KEY(leaf, key);
   5928 					return rc;
   5929 				}
   5930 			}
   5931 		} else {
   5932 			mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
   5933 			if (op == MDB_NEXT_DUP)
   5934 				return MDB_NOTFOUND;
   5935 		}
   5936 	}
   5937 
   5938 	DPRINTF(("cursor_next: top page is %"Z"u in cursor %p",
   5939 		mdb_dbg_pgno(mp), (void *) mc));
   5940 	if (mc->mc_flags & C_DEL) {
   5941 		mc->mc_flags ^= C_DEL;
   5942 		goto skip;
   5943 	}
   5944 
   5945 	if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) {
   5946 		DPUTS("=====> move to next sibling page");
   5947 		if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) {
   5948 			mc->mc_flags |= C_EOF;
   5949 			return rc;
   5950 		}
   5951 		mp = mc->mc_pg[mc->mc_top];
   5952 		DPRINTF(("next page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]));
   5953 	} else
   5954 		mc->mc_ki[mc->mc_top]++;
   5955 
   5956 skip:
   5957 	DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u",
   5958 	    mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top]));
   5959 
   5960 	if (IS_LEAF2(mp)) {
   5961 		key->mv_size = mc->mc_db->md_pad;
   5962 		key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size);
   5963 		return MDB_SUCCESS;
   5964 	}
   5965 
   5966 	mdb_cassert(mc, IS_LEAF(mp));
   5967 	leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
   5968 
   5969 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   5970 		mdb_xcursor_init1(mc, leaf);
   5971 		rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
   5972 		if (rc != MDB_SUCCESS)
   5973 			return rc;
   5974 	} else if (data) {
   5975 		if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
   5976 			return rc;
   5977 	}
   5978 
   5979 	MDB_GET_KEY(leaf, key);
   5980 	return MDB_SUCCESS;
   5981 }
   5982 
   5983 /** Move the cursor to the previous data item. */
   5984 static int
   5985 mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
   5986 {
   5987 	MDB_page	*mp;
   5988 	MDB_node	*leaf;
   5989 	int rc;
   5990 
   5991 	if (!(mc->mc_flags & C_INITIALIZED)) {
   5992 		rc = mdb_cursor_last(mc, key, data);
   5993 		if (rc)
   5994 			return rc;
   5995 		mc->mc_ki[mc->mc_top]++;
   5996 	}
   5997 
   5998 	mp = mc->mc_pg[mc->mc_top];
   5999 
   6000 	if ((mc->mc_db->md_flags & MDB_DUPSORT) &&
   6001 		mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) {
   6002 		leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
   6003 		if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   6004 			if (op == MDB_PREV || op == MDB_PREV_DUP) {
   6005 				rc = mdb_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_PREV);
   6006 				if (op != MDB_PREV || rc != MDB_NOTFOUND) {
   6007 					if (rc == MDB_SUCCESS) {
   6008 						MDB_GET_KEY(leaf, key);
   6009 						mc->mc_flags &= ~C_EOF;
   6010 					}
   6011 					return rc;
   6012 				}
   6013 			}
   6014 		} else {
   6015 			mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
   6016 			if (op == MDB_PREV_DUP)
   6017 				return MDB_NOTFOUND;
   6018 		}
   6019 	}
   6020 
   6021 	DPRINTF(("cursor_prev: top page is %"Z"u in cursor %p",
   6022 		mdb_dbg_pgno(mp), (void *) mc));
   6023 
   6024 	mc->mc_flags &= ~(C_EOF|C_DEL);
   6025 
   6026 	if (mc->mc_ki[mc->mc_top] == 0)  {
   6027 		DPUTS("=====> move to prev sibling page");
   6028 		if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) {
   6029 			return rc;
   6030 		}
   6031 		mp = mc->mc_pg[mc->mc_top];
   6032 		mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1;
   6033 		DPRINTF(("prev page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]));
   6034 	} else
   6035 		mc->mc_ki[mc->mc_top]--;
   6036 
   6037 	DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u",
   6038 	    mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top]));
   6039 
   6040 	if (!IS_LEAF(mp))
   6041 		return MDB_CORRUPTED;
   6042 
   6043 	if (IS_LEAF2(mp)) {
   6044 		key->mv_size = mc->mc_db->md_pad;
   6045 		key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size);
   6046 		return MDB_SUCCESS;
   6047 	}
   6048 
   6049 	leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
   6050 
   6051 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   6052 		mdb_xcursor_init1(mc, leaf);
   6053 		rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL);
   6054 		if (rc != MDB_SUCCESS)
   6055 			return rc;
   6056 	} else if (data) {
   6057 		if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
   6058 			return rc;
   6059 	}
   6060 
   6061 	MDB_GET_KEY(leaf, key);
   6062 	return MDB_SUCCESS;
   6063 }
   6064 
   6065 /** Set the cursor on a specific data item. */
   6066 static int
   6067 mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data,
   6068     MDB_cursor_op op, int *exactp)
   6069 {
   6070 	int		 rc;
   6071 	MDB_page	*mp;
   6072 	MDB_node	*leaf = NULL;
   6073 	DKBUF;
   6074 
   6075 	if (key->mv_size == 0)
   6076 		return MDB_BAD_VALSIZE;
   6077 
   6078 	if (mc->mc_xcursor)
   6079 		mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
   6080 
   6081 	/* See if we're already on the right page */
   6082 	if (mc->mc_flags & C_INITIALIZED) {
   6083 		MDB_val nodekey;
   6084 
   6085 		mp = mc->mc_pg[mc->mc_top];
   6086 		if (!NUMKEYS(mp)) {
   6087 			mc->mc_ki[mc->mc_top] = 0;
   6088 			return MDB_NOTFOUND;
   6089 		}
   6090 		if (MP_FLAGS(mp) & P_LEAF2) {
   6091 			nodekey.mv_size = mc->mc_db->md_pad;
   6092 			nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size);
   6093 		} else {
   6094 			leaf = NODEPTR(mp, 0);
   6095 			MDB_GET_KEY2(leaf, nodekey);
   6096 		}
   6097 		rc = mc->mc_dbx->md_cmp(key, &nodekey);
   6098 		if (rc == 0) {
   6099 			/* Probably happens rarely, but first node on the page
   6100 			 * was the one we wanted.
   6101 			 */
   6102 			mc->mc_ki[mc->mc_top] = 0;
   6103 			if (exactp)
   6104 				*exactp = 1;
   6105 			goto set1;
   6106 		}
   6107 		if (rc > 0) {
   6108 			unsigned int i;
   6109 			unsigned int nkeys = NUMKEYS(mp);
   6110 			if (nkeys > 1) {
   6111 				if (MP_FLAGS(mp) & P_LEAF2) {
   6112 					nodekey.mv_data = LEAF2KEY(mp,
   6113 						 nkeys-1, nodekey.mv_size);
   6114 				} else {
   6115 					leaf = NODEPTR(mp, nkeys-1);
   6116 					MDB_GET_KEY2(leaf, nodekey);
   6117 				}
   6118 				rc = mc->mc_dbx->md_cmp(key, &nodekey);
   6119 				if (rc == 0) {
   6120 					/* last node was the one we wanted */
   6121 					mc->mc_ki[mc->mc_top] = nkeys-1;
   6122 					if (exactp)
   6123 						*exactp = 1;
   6124 					goto set1;
   6125 				}
   6126 				if (rc < 0) {
   6127 					if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) {
   6128 						/* This is definitely the right page, skip search_page */
   6129 						if (MP_FLAGS(mp) & P_LEAF2) {
   6130 							nodekey.mv_data = LEAF2KEY(mp,
   6131 								 mc->mc_ki[mc->mc_top], nodekey.mv_size);
   6132 						} else {
   6133 							leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
   6134 							MDB_GET_KEY2(leaf, nodekey);
   6135 						}
   6136 						rc = mc->mc_dbx->md_cmp(key, &nodekey);
   6137 						if (rc == 0) {
   6138 							/* current node was the one we wanted */
   6139 							if (exactp)
   6140 								*exactp = 1;
   6141 							goto set1;
   6142 						}
   6143 					}
   6144 					rc = 0;
   6145 					mc->mc_flags &= ~C_EOF;
   6146 					goto set2;
   6147 				}
   6148 			}
   6149 			/* If any parents have right-sibs, search.
   6150 			 * Otherwise, there's nothing further.
   6151 			 */
   6152 			for (i=0; i<mc->mc_top; i++)
   6153 				if (mc->mc_ki[i] <
   6154 					NUMKEYS(mc->mc_pg[i])-1)
   6155 					break;
   6156 			if (i == mc->mc_top) {
   6157 				/* There are no other pages */
   6158 				mc->mc_ki[mc->mc_top] = nkeys;
   6159 				return MDB_NOTFOUND;
   6160 			}
   6161 		}
   6162 		if (!mc->mc_top) {
   6163 			/* There are no other pages */
   6164 			mc->mc_ki[mc->mc_top] = 0;
   6165 			if (op == MDB_SET_RANGE && !exactp) {
   6166 				rc = 0;
   6167 				goto set1;
   6168 			} else
   6169 				return MDB_NOTFOUND;
   6170 		}
   6171 	} else {
   6172 		mc->mc_pg[0] = 0;
   6173 	}
   6174 
   6175 	rc = mdb_page_search(mc, key, 0);
   6176 	if (rc != MDB_SUCCESS)
   6177 		return rc;
   6178 
   6179 	mp = mc->mc_pg[mc->mc_top];
   6180 	mdb_cassert(mc, IS_LEAF(mp));
   6181 
   6182 set2:
   6183 	leaf = mdb_node_search(mc, key, exactp);
   6184 	if (exactp != NULL && !*exactp) {
   6185 		/* MDB_SET specified and not an exact match. */
   6186 		return MDB_NOTFOUND;
   6187 	}
   6188 
   6189 	if (leaf == NULL) {
   6190 		DPUTS("===> inexact leaf not found, goto sibling");
   6191 		if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) {
   6192 			mc->mc_flags |= C_EOF;
   6193 			return rc;		/* no entries matched */
   6194 		}
   6195 		mp = mc->mc_pg[mc->mc_top];
   6196 		mdb_cassert(mc, IS_LEAF(mp));
   6197 		leaf = NODEPTR(mp, 0);
   6198 	}
   6199 
   6200 set1:
   6201 	mc->mc_flags |= C_INITIALIZED;
   6202 	mc->mc_flags &= ~C_EOF;
   6203 
   6204 	if (IS_LEAF2(mp)) {
   6205 		if (op == MDB_SET_RANGE || op == MDB_SET_KEY) {
   6206 			key->mv_size = mc->mc_db->md_pad;
   6207 			key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size);
   6208 		}
   6209 		return MDB_SUCCESS;
   6210 	}
   6211 
   6212 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   6213 		mdb_xcursor_init1(mc, leaf);
   6214 		if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) {
   6215 			rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
   6216 		} else {
   6217 			int ex2, *ex2p;
   6218 			if (op == MDB_GET_BOTH) {
   6219 				ex2p = &ex2;
   6220 				ex2 = 0;
   6221 			} else {
   6222 				ex2p = NULL;
   6223 			}
   6224 			rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p);
   6225 			if (rc != MDB_SUCCESS)
   6226 				return rc;
   6227 		}
   6228 	} else if (data) {
   6229 		if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) {
   6230 			MDB_val olddata;
   6231 			MDB_cmp_func *dcmp;
   6232 			if ((rc = mdb_node_read(mc, leaf, &olddata)) != MDB_SUCCESS)
   6233 				return rc;
   6234 			dcmp = mc->mc_dbx->md_dcmp;
   6235 #if UINT_MAX < SIZE_MAX
   6236 			if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
   6237 				dcmp = mdb_cmp_clong;
   6238 #endif
   6239 			rc = dcmp(data, &olddata);
   6240 			if (rc) {
   6241 				if (op == MDB_GET_BOTH || rc > 0)
   6242 					return MDB_NOTFOUND;
   6243 				rc = 0;
   6244 			}
   6245 			*data = olddata;
   6246 
   6247 		} else {
   6248 			if (mc->mc_xcursor)
   6249 				mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
   6250 			if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
   6251 				return rc;
   6252 		}
   6253 	}
   6254 
   6255 	/* The key already matches in all other cases */
   6256 	if (op == MDB_SET_RANGE || op == MDB_SET_KEY)
   6257 		MDB_GET_KEY(leaf, key);
   6258 	DPRINTF(("==> cursor placed on key [%s]", DKEY(key)));
   6259 
   6260 	return rc;
   6261 }
   6262 
   6263 /** Move the cursor to the first item in the database. */
   6264 static int
   6265 mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data)
   6266 {
   6267 	int		 rc;
   6268 	MDB_node	*leaf;
   6269 
   6270 	if (mc->mc_xcursor)
   6271 		mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
   6272 
   6273 	if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
   6274 		rc = mdb_page_search(mc, NULL, MDB_PS_FIRST);
   6275 		if (rc != MDB_SUCCESS)
   6276 			return rc;
   6277 	}
   6278 	mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
   6279 
   6280 	leaf = NODEPTR(mc->mc_pg[mc->mc_top], 0);
   6281 	mc->mc_flags |= C_INITIALIZED;
   6282 	mc->mc_flags &= ~C_EOF;
   6283 
   6284 	mc->mc_ki[mc->mc_top] = 0;
   6285 
   6286 	if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
   6287 		if ( key ) {
   6288 			key->mv_size = mc->mc_db->md_pad;
   6289 			key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size);
   6290 		}
   6291 		return MDB_SUCCESS;
   6292 	}
   6293 
   6294 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   6295 		mdb_xcursor_init1(mc, leaf);
   6296 		rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
   6297 		if (rc)
   6298 			return rc;
   6299 	} else if (data) {
   6300 		if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
   6301 			return rc;
   6302 	}
   6303 
   6304 	MDB_GET_KEY(leaf, key);
   6305 	return MDB_SUCCESS;
   6306 }
   6307 
   6308 /** Move the cursor to the last item in the database. */
   6309 static int
   6310 mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data)
   6311 {
   6312 	int		 rc;
   6313 	MDB_node	*leaf;
   6314 
   6315 	if (mc->mc_xcursor)
   6316 		mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
   6317 
   6318 	if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
   6319 		rc = mdb_page_search(mc, NULL, MDB_PS_LAST);
   6320 		if (rc != MDB_SUCCESS)
   6321 			return rc;
   6322 	}
   6323 	mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
   6324 
   6325 	mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1;
   6326 	mc->mc_flags |= C_INITIALIZED|C_EOF;
   6327 	leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
   6328 
   6329 	if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
   6330 		if (key) {
   6331 			key->mv_size = mc->mc_db->md_pad;
   6332 			key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size);
   6333 		}
   6334 		return MDB_SUCCESS;
   6335 	}
   6336 
   6337 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   6338 		mdb_xcursor_init1(mc, leaf);
   6339 		rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL);
   6340 		if (rc)
   6341 			return rc;
   6342 	} else if (data) {
   6343 		if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
   6344 			return rc;
   6345 	}
   6346 
   6347 	MDB_GET_KEY(leaf, key);
   6348 	return MDB_SUCCESS;
   6349 }
   6350 
   6351 int
   6352 mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data,
   6353     MDB_cursor_op op)
   6354 {
   6355 	int		 rc;
   6356 	int		 exact = 0;
   6357 	int		 (*mfunc)(MDB_cursor *mc, MDB_val *key, MDB_val *data);
   6358 
   6359 	if (mc == NULL)
   6360 		return EINVAL;
   6361 
   6362 	if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED)
   6363 		return MDB_BAD_TXN;
   6364 
   6365 	switch (op) {
   6366 	case MDB_GET_CURRENT:
   6367 		if (!(mc->mc_flags & C_INITIALIZED)) {
   6368 			rc = EINVAL;
   6369 		} else {
   6370 			MDB_page *mp = mc->mc_pg[mc->mc_top];
   6371 			int nkeys = NUMKEYS(mp);
   6372 			if (!nkeys || mc->mc_ki[mc->mc_top] >= nkeys) {
   6373 				mc->mc_ki[mc->mc_top] = nkeys;
   6374 				rc = MDB_NOTFOUND;
   6375 				break;
   6376 			}
   6377 			rc = MDB_SUCCESS;
   6378 			if (IS_LEAF2(mp)) {
   6379 				key->mv_size = mc->mc_db->md_pad;
   6380 				key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size);
   6381 			} else {
   6382 				MDB_node *leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
   6383 				MDB_GET_KEY(leaf, key);
   6384 				if (data) {
   6385 					if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   6386 						rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT);
   6387 					} else {
   6388 						rc = mdb_node_read(mc, leaf, data);
   6389 					}
   6390 				}
   6391 			}
   6392 		}
   6393 		break;
   6394 	case MDB_GET_BOTH:
   6395 	case MDB_GET_BOTH_RANGE:
   6396 		if (data == NULL) {
   6397 			rc = EINVAL;
   6398 			break;
   6399 		}
   6400 		if (mc->mc_xcursor == NULL) {
   6401 			rc = MDB_INCOMPATIBLE;
   6402 			break;
   6403 		}
   6404 		/* FALLTHRU */
   6405 	case MDB_SET:
   6406 	case MDB_SET_KEY:
   6407 	case MDB_SET_RANGE:
   6408 		if (key == NULL) {
   6409 			rc = EINVAL;
   6410 		} else {
   6411 			rc = mdb_cursor_set(mc, key, data, op,
   6412 				op == MDB_SET_RANGE ? NULL : &exact);
   6413 		}
   6414 		break;
   6415 	case MDB_GET_MULTIPLE:
   6416 		if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) {
   6417 			rc = EINVAL;
   6418 			break;
   6419 		}
   6420 		if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) {
   6421 			rc = MDB_INCOMPATIBLE;
   6422 			break;
   6423 		}
   6424 		rc = MDB_SUCCESS;
   6425 		if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) ||
   6426 			(mc->mc_xcursor->mx_cursor.mc_flags & C_EOF))
   6427 			break;
   6428 		goto fetchm;
   6429 	case MDB_NEXT_MULTIPLE:
   6430 		if (data == NULL) {
   6431 			rc = EINVAL;
   6432 			break;
   6433 		}
   6434 		if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) {
   6435 			rc = MDB_INCOMPATIBLE;
   6436 			break;
   6437 		}
   6438 		rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP);
   6439 		if (rc == MDB_SUCCESS) {
   6440 			if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
   6441 				MDB_cursor *mx;
   6442 fetchm:
   6443 				mx = &mc->mc_xcursor->mx_cursor;
   6444 				data->mv_size = NUMKEYS(mx->mc_pg[mx->mc_top]) *
   6445 					mx->mc_db->md_pad;
   6446 				data->mv_data = METADATA(mx->mc_pg[mx->mc_top]);
   6447 				mx->mc_ki[mx->mc_top] = NUMKEYS(mx->mc_pg[mx->mc_top])-1;
   6448 			} else {
   6449 				rc = MDB_NOTFOUND;
   6450 			}
   6451 		}
   6452 		break;
   6453 	case MDB_PREV_MULTIPLE:
   6454 		if (data == NULL) {
   6455 			rc = EINVAL;
   6456 			break;
   6457 		}
   6458 		if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) {
   6459 			rc = MDB_INCOMPATIBLE;
   6460 			break;
   6461 		}
   6462 		if (!(mc->mc_flags & C_INITIALIZED))
   6463 			rc = mdb_cursor_last(mc, key, data);
   6464 		else
   6465 			rc = MDB_SUCCESS;
   6466 		if (rc == MDB_SUCCESS) {
   6467 			MDB_cursor *mx = &mc->mc_xcursor->mx_cursor;
   6468 			if (mx->mc_flags & C_INITIALIZED) {
   6469 				rc = mdb_cursor_sibling(mx, 0);
   6470 				if (rc == MDB_SUCCESS)
   6471 					goto fetchm;
   6472 			} else {
   6473 				rc = MDB_NOTFOUND;
   6474 			}
   6475 		}
   6476 		break;
   6477 	case MDB_NEXT:
   6478 	case MDB_NEXT_DUP:
   6479 	case MDB_NEXT_NODUP:
   6480 		rc = mdb_cursor_next(mc, key, data, op);
   6481 		break;
   6482 	case MDB_PREV:
   6483 	case MDB_PREV_DUP:
   6484 	case MDB_PREV_NODUP:
   6485 		rc = mdb_cursor_prev(mc, key, data, op);
   6486 		break;
   6487 	case MDB_FIRST:
   6488 		rc = mdb_cursor_first(mc, key, data);
   6489 		break;
   6490 	case MDB_FIRST_DUP:
   6491 		mfunc = mdb_cursor_first;
   6492 	mmove:
   6493 		if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) {
   6494 			rc = EINVAL;
   6495 			break;
   6496 		}
   6497 		if (mc->mc_xcursor == NULL) {
   6498 			rc = MDB_INCOMPATIBLE;
   6499 			break;
   6500 		}
   6501 		if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) {
   6502 			mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]);
   6503 			rc = MDB_NOTFOUND;
   6504 			break;
   6505 		}
   6506 		mc->mc_flags &= ~C_EOF;
   6507 		{
   6508 			MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
   6509 			if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   6510 				MDB_GET_KEY(leaf, key);
   6511 				rc = mdb_node_read(mc, leaf, data);
   6512 				break;
   6513 			}
   6514 		}
   6515 		if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
   6516 			rc = EINVAL;
   6517 			break;
   6518 		}
   6519 		rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL);
   6520 		break;
   6521 	case MDB_LAST:
   6522 		rc = mdb_cursor_last(mc, key, data);
   6523 		break;
   6524 	case MDB_LAST_DUP:
   6525 		mfunc = mdb_cursor_last;
   6526 		goto mmove;
   6527 	default:
   6528 		DPRINTF(("unhandled/unimplemented cursor operation %u", op));
   6529 		rc = EINVAL;
   6530 		break;
   6531 	}
   6532 
   6533 	if (mc->mc_flags & C_DEL)
   6534 		mc->mc_flags ^= C_DEL;
   6535 
   6536 	return rc;
   6537 }
   6538 
   6539 /** Touch all the pages in the cursor stack. Set mc_top.
   6540  *	Makes sure all the pages are writable, before attempting a write operation.
   6541  * @param[in] mc The cursor to operate on.
   6542  */
   6543 static int
   6544 mdb_cursor_touch(MDB_cursor *mc)
   6545 {
   6546 	int rc = MDB_SUCCESS;
   6547 
   6548 	if (mc->mc_dbi >= CORE_DBS && !(*mc->mc_dbflag & (DB_DIRTY|DB_DUPDATA))) {
   6549 		/* Touch DB record of named DB */
   6550 		MDB_cursor mc2;
   6551 		MDB_xcursor mcx;
   6552 		if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi))
   6553 			return MDB_BAD_DBI;
   6554 		mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx);
   6555 		rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY);
   6556 		if (rc)
   6557 			 return rc;
   6558 		*mc->mc_dbflag |= DB_DIRTY;
   6559 	}
   6560 	mc->mc_top = 0;
   6561 	if (mc->mc_snum) {
   6562 		do {
   6563 			rc = mdb_page_touch(mc);
   6564 		} while (!rc && ++(mc->mc_top) < mc->mc_snum);
   6565 		mc->mc_top = mc->mc_snum-1;
   6566 	}
   6567 	return rc;
   6568 }
   6569 
   6570 /** Do not spill pages to disk if txn is getting full, may fail instead */
   6571 #define MDB_NOSPILL	0x8000
   6572 
   6573 int
   6574 mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
   6575     unsigned int flags)
   6576 {
   6577 	MDB_env		*env;
   6578 	MDB_node	*leaf = NULL;
   6579 	MDB_page	*fp, *mp, *sub_root = NULL;
   6580 	uint16_t	fp_flags;
   6581 	MDB_val		xdata, *rdata, dkey, olddata;
   6582 	MDB_db dummy;
   6583 	int do_sub = 0, insert_key, insert_data;
   6584 	unsigned int mcount = 0, dcount = 0, nospill;
   6585 	size_t nsize;
   6586 	int rc, rc2;
   6587 	unsigned int nflags;
   6588 	DKBUF;
   6589 
   6590 	if (mc == NULL || key == NULL)
   6591 		return EINVAL;
   6592 
   6593 	env = mc->mc_txn->mt_env;
   6594 
   6595 	/* Check this first so counter will always be zero on any
   6596 	 * early failures.
   6597 	 */
   6598 	if (flags & MDB_MULTIPLE) {
   6599 		dcount = data[1].mv_size;
   6600 		data[1].mv_size = 0;
   6601 		if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED))
   6602 			return MDB_INCOMPATIBLE;
   6603 	}
   6604 
   6605 	nospill = flags & MDB_NOSPILL;
   6606 	flags &= ~MDB_NOSPILL;
   6607 
   6608 	if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
   6609 		return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
   6610 
   6611 	if (key->mv_size-1 >= ENV_MAXKEY(env))
   6612 		return MDB_BAD_VALSIZE;
   6613 
   6614 #if SIZE_MAX > MAXDATASIZE
   6615 	if (data->mv_size > ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE))
   6616 		return MDB_BAD_VALSIZE;
   6617 #else
   6618 	if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size > ENV_MAXKEY(env))
   6619 		return MDB_BAD_VALSIZE;
   6620 #endif
   6621 
   6622 	DPRINTF(("==> put db %d key [%s], size %"Z"u, data size %"Z"u",
   6623 		DDBI(mc), DKEY(key), key ? key->mv_size : 0, data->mv_size));
   6624 
   6625 	dkey.mv_size = 0;
   6626 
   6627 	if (flags & MDB_CURRENT) {
   6628 		if (!(mc->mc_flags & C_INITIALIZED))
   6629 			return EINVAL;
   6630 		rc = MDB_SUCCESS;
   6631 	} else if (mc->mc_db->md_root == P_INVALID) {
   6632 		/* new database, cursor has nothing to point to */
   6633 		mc->mc_snum = 0;
   6634 		mc->mc_top = 0;
   6635 		mc->mc_flags &= ~C_INITIALIZED;
   6636 		rc = MDB_NO_ROOT;
   6637 	} else {
   6638 		int exact = 0;
   6639 		MDB_val d2;
   6640 		if (flags & MDB_APPEND) {
   6641 			MDB_val k2;
   6642 			rc = mdb_cursor_last(mc, &k2, &d2);
   6643 			if (rc == 0) {
   6644 				rc = mc->mc_dbx->md_cmp(key, &k2);
   6645 				if (rc > 0) {
   6646 					rc = MDB_NOTFOUND;
   6647 					mc->mc_ki[mc->mc_top]++;
   6648 				} else {
   6649 					/* new key is <= last key */
   6650 					rc = MDB_KEYEXIST;
   6651 				}
   6652 			}
   6653 		} else {
   6654 			rc = mdb_cursor_set(mc, key, &d2, MDB_SET, &exact);
   6655 		}
   6656 		if ((flags & MDB_NOOVERWRITE) && rc == 0) {
   6657 			DPRINTF(("duplicate key [%s]", DKEY(key)));
   6658 			*data = d2;
   6659 			return MDB_KEYEXIST;
   6660 		}
   6661 		if (rc && rc != MDB_NOTFOUND)
   6662 			return rc;
   6663 	}
   6664 
   6665 	if (mc->mc_flags & C_DEL)
   6666 		mc->mc_flags ^= C_DEL;
   6667 
   6668 	/* Cursor is positioned, check for room in the dirty list */
   6669 	if (!nospill) {
   6670 		if (flags & MDB_MULTIPLE) {
   6671 			rdata = &xdata;
   6672 			xdata.mv_size = data->mv_size * dcount;
   6673 		} else {
   6674 			rdata = data;
   6675 		}
   6676 		if ((rc2 = mdb_page_spill(mc, key, rdata)))
   6677 			return rc2;
   6678 	}
   6679 
   6680 	if (rc == MDB_NO_ROOT) {
   6681 		MDB_page *np;
   6682 		/* new database, write a root leaf page */
   6683 		DPUTS("allocating new root leaf page");
   6684 		if ((rc2 = mdb_page_new(mc, P_LEAF, 1, &np))) {
   6685 			return rc2;
   6686 		}
   6687 		mdb_cursor_push(mc, np);
   6688 		mc->mc_db->md_root = np->mp_pgno;
   6689 		mc->mc_db->md_depth++;
   6690 		*mc->mc_dbflag |= DB_DIRTY;
   6691 		if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED))
   6692 			== MDB_DUPFIXED)
   6693 			MP_FLAGS(np) |= P_LEAF2;
   6694 		mc->mc_flags |= C_INITIALIZED;
   6695 	} else {
   6696 		/* make sure all cursor pages are writable */
   6697 		rc2 = mdb_cursor_touch(mc);
   6698 		if (rc2)
   6699 			return rc2;
   6700 	}
   6701 
   6702 	insert_key = insert_data = rc;
   6703 	if (insert_key) {
   6704 		/* The key does not exist */
   6705 		DPRINTF(("inserting key at index %i", mc->mc_ki[mc->mc_top]));
   6706 		if ((mc->mc_db->md_flags & MDB_DUPSORT) &&
   6707 			LEAFSIZE(key, data) > env->me_nodemax)
   6708 		{
   6709 			/* Too big for a node, insert in sub-DB.  Set up an empty
   6710 			 * "old sub-page" for prep_subDB to expand to a full page.
   6711 			 */
   6712 			fp_flags = P_LEAF|P_DIRTY;
   6713 			fp = env->me_pbuf;
   6714 			fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */
   6715 			MP_LOWER(fp) = MP_UPPER(fp) = (PAGEHDRSZ-PAGEBASE);
   6716 			olddata.mv_size = PAGEHDRSZ;
   6717 			goto prep_subDB;
   6718 		}
   6719 	} else {
   6720 		/* there's only a key anyway, so this is a no-op */
   6721 		if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
   6722 			char *ptr;
   6723 			unsigned int ksize = mc->mc_db->md_pad;
   6724 			if (key->mv_size != ksize)
   6725 				return MDB_BAD_VALSIZE;
   6726 			ptr = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize);
   6727 			memcpy(ptr, key->mv_data, ksize);
   6728 fix_parent:
   6729 			/* if overwriting slot 0 of leaf, need to
   6730 			 * update branch key if there is a parent page
   6731 			 */
   6732 			if (mc->mc_top && !mc->mc_ki[mc->mc_top]) {
   6733 				unsigned short dtop = 1;
   6734 				mc->mc_top--;
   6735 				/* slot 0 is always an empty key, find real slot */
   6736 				while (mc->mc_top && !mc->mc_ki[mc->mc_top]) {
   6737 					mc->mc_top--;
   6738 					dtop++;
   6739 				}
   6740 				if (mc->mc_ki[mc->mc_top])
   6741 					rc2 = mdb_update_key(mc, key);
   6742 				else
   6743 					rc2 = MDB_SUCCESS;
   6744 				mc->mc_top += dtop;
   6745 				if (rc2)
   6746 					return rc2;
   6747 			}
   6748 			return MDB_SUCCESS;
   6749 		}
   6750 
   6751 more:
   6752 		leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
   6753 		olddata.mv_size = NODEDSZ(leaf);
   6754 		olddata.mv_data = NODEDATA(leaf);
   6755 
   6756 		/* DB has dups? */
   6757 		if (F_ISSET(mc->mc_db->md_flags, MDB_DUPSORT)) {
   6758 			/* Prepare (sub-)page/sub-DB to accept the new item,
   6759 			 * if needed.  fp: old sub-page or a header faking
   6760 			 * it.  mp: new (sub-)page.  offset: growth in page
   6761 			 * size.  xdata: node data with new page or DB.
   6762 			 */
   6763 			unsigned	i, offset = 0;
   6764 			mp = fp = xdata.mv_data = env->me_pbuf;
   6765 			mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno;
   6766 
   6767 			/* Was a single item before, must convert now */
   6768 			if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   6769 				MDB_cmp_func *dcmp;
   6770 				/* Just overwrite the current item */
   6771 				if (flags == MDB_CURRENT)
   6772 					goto current;
   6773 				dcmp = mc->mc_dbx->md_dcmp;
   6774 #if UINT_MAX < SIZE_MAX
   6775 				if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
   6776 					dcmp = mdb_cmp_clong;
   6777 #endif
   6778 				/* does data match? */
   6779 				if (!dcmp(data, &olddata)) {
   6780 					if (flags & (MDB_NODUPDATA|MDB_APPENDDUP))
   6781 						return MDB_KEYEXIST;
   6782 					/* overwrite it */
   6783 					goto current;
   6784 				}
   6785 
   6786 				/* Back up original data item */
   6787 				dkey.mv_size = olddata.mv_size;
   6788 				dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size);
   6789 
   6790 				/* Make sub-page header for the dup items, with dummy body */
   6791 				MP_FLAGS(fp) = P_LEAF|P_DIRTY|P_SUBP;
   6792 				MP_LOWER(fp) = (PAGEHDRSZ-PAGEBASE);
   6793 				xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size;
   6794 				if (mc->mc_db->md_flags & MDB_DUPFIXED) {
   6795 					MP_FLAGS(fp) |= P_LEAF2;
   6796 					fp->mp_pad = data->mv_size;
   6797 					xdata.mv_size += 2 * data->mv_size;	/* leave space for 2 more */
   6798 				} else {
   6799 					xdata.mv_size += 2 * (sizeof(indx_t) + NODESIZE) +
   6800 						(dkey.mv_size & 1) + (data->mv_size & 1);
   6801 				}
   6802 				MP_UPPER(fp) = xdata.mv_size - PAGEBASE;
   6803 				olddata.mv_size = xdata.mv_size; /* pretend olddata is fp */
   6804 			} else if (leaf->mn_flags & F_SUBDATA) {
   6805 				/* Data is on sub-DB, just store it */
   6806 				flags |= F_DUPDATA|F_SUBDATA;
   6807 				goto put_sub;
   6808 			} else {
   6809 				/* Data is on sub-page */
   6810 				fp = olddata.mv_data;
   6811 				switch (flags) {
   6812 				default:
   6813 					if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) {
   6814 						offset = EVEN(NODESIZE + sizeof(indx_t) +
   6815 							data->mv_size);
   6816 						break;
   6817 					}
   6818 					offset = fp->mp_pad;
   6819 					if (SIZELEFT(fp) < offset) {
   6820 						offset *= 4; /* space for 4 more */
   6821 						break;
   6822 					}
   6823 					/* FALLTHRU */ /* Big enough MDB_DUPFIXED sub-page */
   6824 				case MDB_CURRENT:
   6825 					MP_FLAGS(fp) |= P_DIRTY;
   6826 					COPY_PGNO(MP_PGNO(fp), MP_PGNO(mp));
   6827 					mc->mc_xcursor->mx_cursor.mc_pg[0] = fp;
   6828 					flags |= F_DUPDATA;
   6829 					goto put_sub;
   6830 				}
   6831 				xdata.mv_size = olddata.mv_size + offset;
   6832 			}
   6833 
   6834 			fp_flags = MP_FLAGS(fp);
   6835 			if (NODESIZE + NODEKSZ(leaf) + xdata.mv_size > env->me_nodemax) {
   6836 					/* Too big for a sub-page, convert to sub-DB */
   6837 					fp_flags &= ~P_SUBP;
   6838 prep_subDB:
   6839 					if (mc->mc_db->md_flags & MDB_DUPFIXED) {
   6840 						fp_flags |= P_LEAF2;
   6841 						dummy.md_pad = fp->mp_pad;
   6842 						dummy.md_flags = MDB_DUPFIXED;
   6843 						if (mc->mc_db->md_flags & MDB_INTEGERDUP)
   6844 							dummy.md_flags |= MDB_INTEGERKEY;
   6845 					} else {
   6846 						dummy.md_pad = 0;
   6847 						dummy.md_flags = 0;
   6848 					}
   6849 					dummy.md_depth = 1;
   6850 					dummy.md_branch_pages = 0;
   6851 					dummy.md_leaf_pages = 1;
   6852 					dummy.md_overflow_pages = 0;
   6853 					dummy.md_entries = NUMKEYS(fp);
   6854 					xdata.mv_size = sizeof(MDB_db);
   6855 					xdata.mv_data = &dummy;
   6856 					if ((rc = mdb_page_alloc(mc, 1, &mp)))
   6857 						return rc;
   6858 					offset = env->me_psize - olddata.mv_size;
   6859 					flags |= F_DUPDATA|F_SUBDATA;
   6860 					dummy.md_root = mp->mp_pgno;
   6861 					sub_root = mp;
   6862 			}
   6863 			if (mp != fp) {
   6864 				MP_FLAGS(mp) = fp_flags | P_DIRTY;
   6865 				MP_PAD(mp)   = MP_PAD(fp);
   6866 				MP_LOWER(mp) = MP_LOWER(fp);
   6867 				MP_UPPER(mp) = MP_UPPER(fp) + offset;
   6868 				if (fp_flags & P_LEAF2) {
   6869 					memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad);
   6870 				} else {
   6871 					memcpy((char *)mp + MP_UPPER(mp) + PAGEBASE, (char *)fp + MP_UPPER(fp) + PAGEBASE,
   6872 						olddata.mv_size - MP_UPPER(fp) - PAGEBASE);
   6873 					memcpy((char *)MP_PTRS(mp), (char *)MP_PTRS(fp), NUMKEYS(fp) * sizeof(mp->mp_ptrs[0]));
   6874 					for (i=0; i<NUMKEYS(fp); i++)
   6875 						mp->mp_ptrs[i] += offset;
   6876 				}
   6877 			}
   6878 
   6879 			rdata = &xdata;
   6880 			flags |= F_DUPDATA;
   6881 			do_sub = 1;
   6882 			if (!insert_key)
   6883 				mdb_node_del(mc, 0);
   6884 			goto new_sub;
   6885 		}
   6886 current:
   6887 		/* LMDB passes F_SUBDATA in 'flags' to write a DB record */
   6888 		if ((leaf->mn_flags ^ flags) & F_SUBDATA)
   6889 			return MDB_INCOMPATIBLE;
   6890 		/* overflow page overwrites need special handling */
   6891 		if (F_ISSET(leaf->mn_flags, F_BIGDATA)) {
   6892 			MDB_page *omp;
   6893 			pgno_t pg;
   6894 			int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize);
   6895 
   6896 			memcpy(&pg, olddata.mv_data, sizeof(pg));
   6897 			if ((rc2 = mdb_page_get(mc, pg, &omp, &level)) != 0)
   6898 				return rc2;
   6899 			ovpages = omp->mp_pages;
   6900 
   6901 			/* Is the ov page large enough? */
   6902 			if (ovpages >= dpages) {
   6903 			  if (!(omp->mp_flags & P_DIRTY) &&
   6904 				  (level || (env->me_flags & MDB_WRITEMAP)))
   6905 			  {
   6906 				rc = mdb_page_unspill(mc->mc_txn, omp, &omp);
   6907 				if (rc)
   6908 					return rc;
   6909 				level = 0;		/* dirty in this txn or clean */
   6910 			  }
   6911 			  /* Is it dirty? */
   6912 			  if (omp->mp_flags & P_DIRTY) {
   6913 				/* yes, overwrite it. Note in this case we don't
   6914 				 * bother to try shrinking the page if the new data
   6915 				 * is smaller than the overflow threshold.
   6916 				 */
   6917 				if (level > 1) {
   6918 					/* It is writable only in a parent txn */
   6919 					size_t sz = (size_t) env->me_psize * ovpages, off;
   6920 					MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages);
   6921 					MDB_ID2 id2;
   6922 					if (!np)
   6923 						return ENOMEM;
   6924 					id2.mid = pg;
   6925 					id2.mptr = np;
   6926 					/* Note - this page is already counted in parent's dirty_room */
   6927 					rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2);
   6928 					mdb_cassert(mc, rc2 == 0);
   6929 					/* Currently we make the page look as with put() in the
   6930 					 * parent txn, in case the user peeks at MDB_RESERVEd
   6931 					 * or unused parts. Some users treat ovpages specially.
   6932 					 */
   6933 					if (!(flags & MDB_RESERVE)) {
   6934 						/* Skip the part where LMDB will put *data.
   6935 						 * Copy end of page, adjusting alignment so
   6936 						 * compiler may copy words instead of bytes.
   6937 						 */
   6938 						off = (PAGEHDRSZ + data->mv_size) & -(int)sizeof(size_t);
   6939 						memcpy((size_t *)((char *)np + off),
   6940 							(size_t *)((char *)omp + off), sz - off);
   6941 						sz = PAGEHDRSZ;
   6942 					}
   6943 					memcpy(np, omp, sz); /* Copy beginning of page */
   6944 					omp = np;
   6945 				}
   6946 				SETDSZ(leaf, data->mv_size);
   6947 				if (F_ISSET(flags, MDB_RESERVE))
   6948 					data->mv_data = METADATA(omp);
   6949 				else
   6950 					memcpy(METADATA(omp), data->mv_data, data->mv_size);
   6951 				return MDB_SUCCESS;
   6952 			  }
   6953 			}
   6954 			if ((rc2 = mdb_ovpage_free(mc, omp)) != MDB_SUCCESS)
   6955 				return rc2;
   6956 		} else if (data->mv_size == olddata.mv_size) {
   6957 			/* same size, just replace it. Note that we could
   6958 			 * also reuse this node if the new data is smaller,
   6959 			 * but instead we opt to shrink the node in that case.
   6960 			 */
   6961 			if (F_ISSET(flags, MDB_RESERVE))
   6962 				data->mv_data = olddata.mv_data;
   6963 			else if (!(mc->mc_flags & C_SUB))
   6964 				memcpy(olddata.mv_data, data->mv_data, data->mv_size);
   6965 			else {
   6966 				if (key->mv_size != NODEKSZ(leaf))
   6967 					goto new_ksize;
   6968 				memcpy(NODEKEY(leaf), key->mv_data, key->mv_size);
   6969 				goto fix_parent;
   6970 			}
   6971 			return MDB_SUCCESS;
   6972 		}
   6973 new_ksize:
   6974 		mdb_node_del(mc, 0);
   6975 	}
   6976 
   6977 	rdata = data;
   6978 
   6979 new_sub:
   6980 	nflags = flags & NODE_ADD_FLAGS;
   6981 	nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->mv_size : mdb_leaf_size(env, key, rdata);
   6982 	if (SIZELEFT(mc->mc_pg[mc->mc_top]) < nsize) {
   6983 		if (( flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA )
   6984 			nflags &= ~MDB_APPEND; /* sub-page may need room to grow */
   6985 		if (!insert_key)
   6986 			nflags |= MDB_SPLIT_REPLACE;
   6987 		rc = mdb_page_split(mc, key, rdata, P_INVALID, nflags);
   6988 	} else {
   6989 		/* There is room already in this leaf page. */
   6990 		rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags);
   6991 		if (rc == 0) {
   6992 			/* Adjust other cursors pointing to mp */
   6993 			MDB_cursor *m2, *m3;
   6994 			MDB_dbi dbi = mc->mc_dbi;
   6995 			unsigned i = mc->mc_top;
   6996 			MDB_page *mp = mc->mc_pg[i];
   6997 
   6998 			for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
   6999 				if (mc->mc_flags & C_SUB)
   7000 					m3 = &m2->mc_xcursor->mx_cursor;
   7001 				else
   7002 					m3 = m2;
   7003 				if (m3 == mc || m3->mc_snum < mc->mc_snum || m3->mc_pg[i] != mp) continue;
   7004 				if (m3->mc_ki[i] >= mc->mc_ki[i] && insert_key) {
   7005 					m3->mc_ki[i]++;
   7006 				}
   7007 				XCURSOR_REFRESH(m3, i, mp);
   7008 			}
   7009 		}
   7010 	}
   7011 
   7012 	if (rc == MDB_SUCCESS) {
   7013 		/* Now store the actual data in the child DB. Note that we're
   7014 		 * storing the user data in the keys field, so there are strict
   7015 		 * size limits on dupdata. The actual data fields of the child
   7016 		 * DB are all zero size.
   7017 		 */
   7018 		if (do_sub) {
   7019 			int xflags, new_dupdata;
   7020 			size_t ecount;
   7021 put_sub:
   7022 			xdata.mv_size = 0;
   7023 			xdata.mv_data = "";
   7024 			leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
   7025 			if ((flags & (MDB_CURRENT|MDB_APPENDDUP)) == MDB_CURRENT) {
   7026 				xflags = MDB_CURRENT|MDB_NOSPILL;
   7027 			} else {
   7028 				mdb_xcursor_init1(mc, leaf);
   7029 				xflags = (flags & MDB_NODUPDATA) ?
   7030 					MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL;
   7031 			}
   7032 			if (sub_root)
   7033 				mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root;
   7034 			new_dupdata = (int)dkey.mv_size;
   7035 			/* converted, write the original data first */
   7036 			if (dkey.mv_size) {
   7037 				rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags);
   7038 				if (rc)
   7039 					goto bad_sub;
   7040 				/* we've done our job */
   7041 				dkey.mv_size = 0;
   7042 			}
   7043 			if (!(leaf->mn_flags & F_SUBDATA) || sub_root) {
   7044 				/* Adjust other cursors pointing to mp */
   7045 				MDB_cursor *m2;
   7046 				MDB_xcursor *mx = mc->mc_xcursor;
   7047 				unsigned i = mc->mc_top;
   7048 				MDB_page *mp = mc->mc_pg[i];
   7049 
   7050 				for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) {
   7051 					if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
   7052 					if (!(m2->mc_flags & C_INITIALIZED)) continue;
   7053 					if (m2->mc_pg[i] == mp) {
   7054 						if (m2->mc_ki[i] == mc->mc_ki[i]) {
   7055 							mdb_xcursor_init2(m2, mx, new_dupdata);
   7056 						} else if (!insert_key) {
   7057 							XCURSOR_REFRESH(m2, i, mp);
   7058 						}
   7059 					}
   7060 				}
   7061 			}
   7062 			ecount = mc->mc_xcursor->mx_db.md_entries;
   7063 			if (flags & MDB_APPENDDUP)
   7064 				xflags |= MDB_APPEND;
   7065 			rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags);
   7066 			if (flags & F_SUBDATA) {
   7067 				void *db = NODEDATA(leaf);
   7068 				memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db));
   7069 			}
   7070 			insert_data = mc->mc_xcursor->mx_db.md_entries - ecount;
   7071 		}
   7072 		/* Increment count unless we just replaced an existing item. */
   7073 		if (insert_data)
   7074 			mc->mc_db->md_entries++;
   7075 		if (insert_key) {
   7076 			/* Invalidate txn if we created an empty sub-DB */
   7077 			if (rc)
   7078 				goto bad_sub;
   7079 			/* If we succeeded and the key didn't exist before,
   7080 			 * make sure the cursor is marked valid.
   7081 			 */
   7082 			mc->mc_flags |= C_INITIALIZED;
   7083 		}
   7084 		if (flags & MDB_MULTIPLE) {
   7085 			if (!rc) {
   7086 				mcount++;
   7087 				/* let caller know how many succeeded, if any */
   7088 				data[1].mv_size = mcount;
   7089 				if (mcount < dcount) {
   7090 					data[0].mv_data = (char *)data[0].mv_data + data[0].mv_size;
   7091 					insert_key = insert_data = 0;
   7092 					goto more;
   7093 				}
   7094 			}
   7095 		}
   7096 		return rc;
   7097 bad_sub:
   7098 		if (rc == MDB_KEYEXIST)	/* should not happen, we deleted that item */
   7099 			rc = MDB_CORRUPTED;
   7100 	}
   7101 	mc->mc_txn->mt_flags |= MDB_TXN_ERROR;
   7102 	return rc;
   7103 }
   7104 
   7105 int
   7106 mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
   7107 {
   7108 	MDB_node	*leaf;
   7109 	MDB_page	*mp;
   7110 	int rc;
   7111 
   7112 	if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
   7113 		return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
   7114 
   7115 	if (!(mc->mc_flags & C_INITIALIZED))
   7116 		return EINVAL;
   7117 
   7118 	if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top]))
   7119 		return MDB_NOTFOUND;
   7120 
   7121 	if (!(flags & MDB_NOSPILL) && (rc = mdb_page_spill(mc, NULL, NULL)))
   7122 		return rc;
   7123 
   7124 	rc = mdb_cursor_touch(mc);
   7125 	if (rc)
   7126 		return rc;
   7127 
   7128 	mp = mc->mc_pg[mc->mc_top];
   7129 	if (!IS_LEAF(mp))
   7130 		return MDB_CORRUPTED;
   7131 	if (IS_LEAF2(mp))
   7132 		goto del_key;
   7133 	leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
   7134 
   7135 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   7136 		if (flags & MDB_NODUPDATA) {
   7137 			/* mdb_cursor_del0() will subtract the final entry */
   7138 			mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1;
   7139 			mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
   7140 		} else {
   7141 			if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) {
   7142 				mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf);
   7143 			}
   7144 			rc = mdb_cursor_del(&mc->mc_xcursor->mx_cursor, MDB_NOSPILL);
   7145 			if (rc)
   7146 				return rc;
   7147 			/* If sub-DB still has entries, we're done */
   7148 			if (mc->mc_xcursor->mx_db.md_entries) {
   7149 				if (leaf->mn_flags & F_SUBDATA) {
   7150 					/* update subDB info */
   7151 					void *db = NODEDATA(leaf);
   7152 					memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db));
   7153 				} else {
   7154 					MDB_cursor *m2;
   7155 					/* shrink fake page */
   7156 					mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]);
   7157 					leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
   7158 					mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf);
   7159 					/* fix other sub-DB cursors pointed at fake pages on this page */
   7160 					for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) {
   7161 						if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
   7162 						if (!(m2->mc_flags & C_INITIALIZED)) continue;
   7163 						if (m2->mc_pg[mc->mc_top] == mp) {
   7164 							XCURSOR_REFRESH(m2, mc->mc_top, mp);
   7165 						}
   7166 					}
   7167 				}
   7168 				mc->mc_db->md_entries--;
   7169 				return rc;
   7170 			} else {
   7171 				mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
   7172 			}
   7173 			/* otherwise fall thru and delete the sub-DB */
   7174 		}
   7175 
   7176 		if (leaf->mn_flags & F_SUBDATA) {
   7177 			/* add all the child DB's pages to the free list */
   7178 			rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0);
   7179 			if (rc)
   7180 				goto fail;
   7181 		}
   7182 	}
   7183 	/* LMDB passes F_SUBDATA in 'flags' to delete a DB record */
   7184 	else if ((leaf->mn_flags ^ flags) & F_SUBDATA) {
   7185 		rc = MDB_INCOMPATIBLE;
   7186 		goto fail;
   7187 	}
   7188 
   7189 	/* add overflow pages to free list */
   7190 	if (F_ISSET(leaf->mn_flags, F_BIGDATA)) {
   7191 		MDB_page *omp;
   7192 		pgno_t pg;
   7193 
   7194 		memcpy(&pg, NODEDATA(leaf), sizeof(pg));
   7195 		if ((rc = mdb_page_get(mc, pg, &omp, NULL)) ||
   7196 			(rc = mdb_ovpage_free(mc, omp)))
   7197 			goto fail;
   7198 	}
   7199 
   7200 del_key:
   7201 	return mdb_cursor_del0(mc);
   7202 
   7203 fail:
   7204 	mc->mc_txn->mt_flags |= MDB_TXN_ERROR;
   7205 	return rc;
   7206 }
   7207 
   7208 /** Allocate and initialize new pages for a database.
   7209  * Set #MDB_TXN_ERROR on failure.
   7210  * @param[in] mc a cursor on the database being added to.
   7211  * @param[in] flags flags defining what type of page is being allocated.
   7212  * @param[in] num the number of pages to allocate. This is usually 1,
   7213  * unless allocating overflow pages for a large record.
   7214  * @param[out] mp Address of a page, or NULL on failure.
   7215  * @return 0 on success, non-zero on failure.
   7216  */
   7217 static int
   7218 mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp)
   7219 {
   7220 	MDB_page	*np;
   7221 	int rc;
   7222 
   7223 	if ((rc = mdb_page_alloc(mc, num, &np)))
   7224 		return rc;
   7225 	DPRINTF(("allocated new mpage %"Z"u, page size %u",
   7226 	    np->mp_pgno, mc->mc_txn->mt_env->me_psize));
   7227 	np->mp_flags = flags | P_DIRTY;
   7228 	np->mp_lower = (PAGEHDRSZ-PAGEBASE);
   7229 	np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE;
   7230 
   7231 	if (IS_BRANCH(np))
   7232 		mc->mc_db->md_branch_pages++;
   7233 	else if (IS_LEAF(np))
   7234 		mc->mc_db->md_leaf_pages++;
   7235 	else if (IS_OVERFLOW(np)) {
   7236 		mc->mc_db->md_overflow_pages += num;
   7237 		np->mp_pages = num;
   7238 	}
   7239 	*mp = np;
   7240 
   7241 	return 0;
   7242 }
   7243 
   7244 /** Calculate the size of a leaf node.
   7245  * The size depends on the environment's page size; if a data item
   7246  * is too large it will be put onto an overflow page and the node
   7247  * size will only include the key and not the data. Sizes are always
   7248  * rounded up to an even number of bytes, to guarantee 2-byte alignment
   7249  * of the #MDB_node headers.
   7250  * @param[in] env The environment handle.
   7251  * @param[in] key The key for the node.
   7252  * @param[in] data The data for the node.
   7253  * @return The number of bytes needed to store the node.
   7254  */
   7255 static size_t
   7256 mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data)
   7257 {
   7258 	size_t		 sz;
   7259 
   7260 	sz = LEAFSIZE(key, data);
   7261 	if (sz > env->me_nodemax) {
   7262 		/* put on overflow page */
   7263 		sz -= data->mv_size - sizeof(pgno_t);
   7264 	}
   7265 
   7266 	return EVEN(sz + sizeof(indx_t));
   7267 }
   7268 
   7269 /** Calculate the size of a branch node.
   7270  * The size should depend on the environment's page size but since
   7271  * we currently don't support spilling large keys onto overflow
   7272  * pages, it's simply the size of the #MDB_node header plus the
   7273  * size of the key. Sizes are always rounded up to an even number
   7274  * of bytes, to guarantee 2-byte alignment of the #MDB_node headers.
   7275  * @param[in] env The environment handle.
   7276  * @param[in] key The key for the node.
   7277  * @return The number of bytes needed to store the node.
   7278  */
   7279 static size_t
   7280 mdb_branch_size(MDB_env *env, MDB_val *key)
   7281 {
   7282 	size_t		 sz;
   7283 
   7284 	sz = INDXSIZE(key);
   7285 	if (sz > env->me_nodemax) {
   7286 		/* put on overflow page */
   7287 		/* not implemented */
   7288 		/* sz -= key->size - sizeof(pgno_t); */
   7289 	}
   7290 
   7291 	return sz + sizeof(indx_t);
   7292 }
   7293 
   7294 /** Add a node to the page pointed to by the cursor.
   7295  * Set #MDB_TXN_ERROR on failure.
   7296  * @param[in] mc The cursor for this operation.
   7297  * @param[in] indx The index on the page where the new node should be added.
   7298  * @param[in] key The key for the new node.
   7299  * @param[in] data The data for the new node, if any.
   7300  * @param[in] pgno The page number, if adding a branch node.
   7301  * @param[in] flags Flags for the node.
   7302  * @return 0 on success, non-zero on failure. Possible errors are:
   7303  * <ul>
   7304  *	<li>ENOMEM - failed to allocate overflow pages for the node.
   7305  *	<li>MDB_PAGE_FULL - there is insufficient room in the page. This error
   7306  *	should never happen since all callers already calculate the
   7307  *	page's free space before calling this function.
   7308  * </ul>
   7309  */
   7310 static int
   7311 mdb_node_add(MDB_cursor *mc, indx_t indx,
   7312     MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags)
   7313 {
   7314 	unsigned int	 i;
   7315 	size_t		 node_size = NODESIZE;
   7316 	ssize_t		 room;
   7317 	indx_t		 ofs;
   7318 	MDB_node	*node;
   7319 	MDB_page	*mp = mc->mc_pg[mc->mc_top];
   7320 	MDB_page	*ofp = NULL;		/* overflow page */
   7321 	void		*ndata;
   7322 	DKBUF;
   7323 
   7324 	mdb_cassert(mc, MP_UPPER(mp) >= MP_LOWER(mp));
   7325 
   7326 	DPRINTF(("add to %s %spage %"Z"u index %i, data size %"Z"u key size %"Z"u [%s]",
   7327 	    IS_LEAF(mp) ? "leaf" : "branch",
   7328 		IS_SUBP(mp) ? "sub-" : "",
   7329 		mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0,
   7330 		key ? key->mv_size : 0, key ? DKEY(key) : "null"));
   7331 
   7332 	if (IS_LEAF2(mp)) {
   7333 		/* Move higher keys up one slot. */
   7334 		int ksize = mc->mc_db->md_pad, dif;
   7335 		char *ptr = LEAF2KEY(mp, indx, ksize);
   7336 		dif = NUMKEYS(mp) - indx;
   7337 		if (dif > 0)
   7338 			memmove(ptr+ksize, ptr, dif*ksize);
   7339 		/* insert new key */
   7340 		memcpy(ptr, key->mv_data, ksize);
   7341 
   7342 		/* Just using these for counting */
   7343 		MP_LOWER(mp) += sizeof(indx_t);
   7344 		MP_UPPER(mp) -= ksize - sizeof(indx_t);
   7345 		return MDB_SUCCESS;
   7346 	}
   7347 
   7348 	room = (ssize_t)SIZELEFT(mp) - (ssize_t)sizeof(indx_t);
   7349 	if (key != NULL)
   7350 		node_size += key->mv_size;
   7351 	if (IS_LEAF(mp)) {
   7352 		mdb_cassert(mc, key && data);
   7353 		if (F_ISSET(flags, F_BIGDATA)) {
   7354 			/* Data already on overflow page. */
   7355 			node_size += sizeof(pgno_t);
   7356 		} else if (node_size + data->mv_size > mc->mc_txn->mt_env->me_nodemax) {
   7357 			int ovpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize);
   7358 			int rc;
   7359 			/* Put data on overflow page. */
   7360 			DPRINTF(("data size is %"Z"u, node would be %"Z"u, put data on overflow page",
   7361 			    data->mv_size, node_size+data->mv_size));
   7362 			node_size = EVEN(node_size + sizeof(pgno_t));
   7363 			if ((ssize_t)node_size > room)
   7364 				goto full;
   7365 			if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp)))
   7366 				return rc;
   7367 			DPRINTF(("allocated overflow page %"Z"u", ofp->mp_pgno));
   7368 			flags |= F_BIGDATA;
   7369 			goto update;
   7370 		} else {
   7371 			node_size += data->mv_size;
   7372 		}
   7373 	}
   7374 	node_size = EVEN(node_size);
   7375 	if ((ssize_t)node_size > room)
   7376 		goto full;
   7377 
   7378 update:
   7379 	/* Move higher pointers up one slot. */
   7380 	for (i = NUMKEYS(mp); i > indx; i--)
   7381 		MP_PTRS(mp)[i] = MP_PTRS(mp)[i - 1];
   7382 
   7383 	/* Adjust free space offsets. */
   7384 	ofs = MP_UPPER(mp) - node_size;
   7385 	mdb_cassert(mc, ofs >= MP_LOWER(mp) + sizeof(indx_t));
   7386 	MP_PTRS(mp)[indx] = ofs;
   7387 	MP_UPPER(mp) = ofs;
   7388 	MP_LOWER(mp) += sizeof(indx_t);
   7389 
   7390 	/* Write the node data. */
   7391 	node = NODEPTR(mp, indx);
   7392 	node->mn_ksize = (key == NULL) ? 0 : key->mv_size;
   7393 	node->mn_flags = flags;
   7394 	if (IS_LEAF(mp))
   7395 		SETDSZ(node,data->mv_size);
   7396 	else
   7397 		SETPGNO(node,pgno);
   7398 
   7399 	if (key)
   7400 		memcpy(NODEKEY(node), key->mv_data, key->mv_size);
   7401 
   7402 	if (IS_LEAF(mp)) {
   7403 		ndata = NODEDATA(node);
   7404 		if (ofp == NULL) {
   7405 			if (F_ISSET(flags, F_BIGDATA))
   7406 				memcpy(ndata, data->mv_data, sizeof(pgno_t));
   7407 			else if (F_ISSET(flags, MDB_RESERVE))
   7408 				data->mv_data = ndata;
   7409 			else
   7410 				memcpy(ndata, data->mv_data, data->mv_size);
   7411 		} else {
   7412 			memcpy(ndata, &ofp->mp_pgno, sizeof(pgno_t));
   7413 			ndata = METADATA(ofp);
   7414 			if (F_ISSET(flags, MDB_RESERVE))
   7415 				data->mv_data = ndata;
   7416 			else
   7417 				memcpy(ndata, data->mv_data, data->mv_size);
   7418 		}
   7419 	}
   7420 
   7421 	return MDB_SUCCESS;
   7422 
   7423 full:
   7424 	DPRINTF(("not enough room in page %"Z"u, got %u ptrs",
   7425 		mdb_dbg_pgno(mp), NUMKEYS(mp)));
   7426 	DPRINTF(("upper-lower = %u - %u = %"Z"d", MP_UPPER(mp),MP_LOWER(mp),room));
   7427 	DPRINTF(("node size = %"Z"u", node_size));
   7428 	mc->mc_txn->mt_flags |= MDB_TXN_ERROR;
   7429 	return MDB_PAGE_FULL;
   7430 }
   7431 
   7432 /** Delete the specified node from a page.
   7433  * @param[in] mc Cursor pointing to the node to delete.
   7434  * @param[in] ksize The size of a node. Only used if the page is
   7435  * part of a #MDB_DUPFIXED database.
   7436  */
   7437 static void
   7438 mdb_node_del(MDB_cursor *mc, int ksize)
   7439 {
   7440 	MDB_page *mp = mc->mc_pg[mc->mc_top];
   7441 	indx_t	indx = mc->mc_ki[mc->mc_top];
   7442 	unsigned int	 sz;
   7443 	indx_t		 i, j, numkeys, ptr;
   7444 	MDB_node	*node;
   7445 	char		*base;
   7446 
   7447 	DPRINTF(("delete node %u on %s page %"Z"u", indx,
   7448 	    IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp)));
   7449 	numkeys = NUMKEYS(mp);
   7450 	mdb_cassert(mc, indx < numkeys);
   7451 
   7452 	if (IS_LEAF2(mp)) {
   7453 		int x = numkeys - 1 - indx;
   7454 		base = LEAF2KEY(mp, indx, ksize);
   7455 		if (x)
   7456 			memmove(base, base + ksize, x * ksize);
   7457 		MP_LOWER(mp) -= sizeof(indx_t);
   7458 		MP_UPPER(mp) += ksize - sizeof(indx_t);
   7459 		return;
   7460 	}
   7461 
   7462 	node = NODEPTR(mp, indx);
   7463 	sz = NODESIZE + node->mn_ksize;
   7464 	if (IS_LEAF(mp)) {
   7465 		if (F_ISSET(node->mn_flags, F_BIGDATA))
   7466 			sz += sizeof(pgno_t);
   7467 		else
   7468 			sz += NODEDSZ(node);
   7469 	}
   7470 	sz = EVEN(sz);
   7471 
   7472 	ptr = MP_PTRS(mp)[indx];
   7473 	for (i = j = 0; i < numkeys; i++) {
   7474 		if (i != indx) {
   7475 			MP_PTRS(mp)[j] = MP_PTRS(mp)[i];
   7476 			if (MP_PTRS(mp)[i] < ptr)
   7477 				MP_PTRS(mp)[j] += sz;
   7478 			j++;
   7479 		}
   7480 	}
   7481 
   7482 	base = (char *)mp + MP_UPPER(mp) + PAGEBASE;
   7483 	memmove(base + sz, base, ptr - MP_UPPER(mp));
   7484 
   7485 	MP_LOWER(mp) -= sizeof(indx_t);
   7486 	MP_UPPER(mp) += sz;
   7487 }
   7488 
   7489 /** Compact the main page after deleting a node on a subpage.
   7490  * @param[in] mp The main page to operate on.
   7491  * @param[in] indx The index of the subpage on the main page.
   7492  */
   7493 static void
   7494 mdb_node_shrink(MDB_page *mp, indx_t indx)
   7495 {
   7496 	MDB_node *node;
   7497 	MDB_page *sp, *xp;
   7498 	char *base;
   7499 	indx_t delta, nsize, len, ptr;
   7500 	int i;
   7501 
   7502 	node = NODEPTR(mp, indx);
   7503 	sp = (MDB_page *)NODEDATA(node);
   7504 	delta = SIZELEFT(sp);
   7505 	nsize = NODEDSZ(node) - delta;
   7506 
   7507 	/* Prepare to shift upward, set len = length(subpage part to shift) */
   7508 	if (IS_LEAF2(sp)) {
   7509 		len = nsize;
   7510 		if (nsize & 1)
   7511 			return;		/* do not make the node uneven-sized */
   7512 	} else {
   7513 		xp = (MDB_page *)((char *)sp + delta); /* destination subpage */
   7514 		for (i = NUMKEYS(sp); --i >= 0; )
   7515 			MP_PTRS(xp)[i] = MP_PTRS(sp)[i] - delta;
   7516 		len = PAGEHDRSZ;
   7517 	}
   7518 	MP_UPPER(sp) = MP_LOWER(sp);
   7519 	COPY_PGNO(MP_PGNO(sp), mp->mp_pgno);
   7520 	SETDSZ(node, nsize);
   7521 
   7522 	/* Shift <lower nodes...initial part of subpage> upward */
   7523 	base = (char *)mp + mp->mp_upper + PAGEBASE;
   7524 	memmove(base + delta, base, (char *)sp + len - base);
   7525 
   7526 	ptr = mp->mp_ptrs[indx];
   7527 	for (i = NUMKEYS(mp); --i >= 0; ) {
   7528 		if (mp->mp_ptrs[i] <= ptr)
   7529 			mp->mp_ptrs[i] += delta;
   7530 	}
   7531 	mp->mp_upper += delta;
   7532 }
   7533 
   7534 /** Initial setup of a sorted-dups cursor.
   7535  * Sorted duplicates are implemented as a sub-database for the given key.
   7536  * The duplicate data items are actually keys of the sub-database.
   7537  * Operations on the duplicate data items are performed using a sub-cursor
   7538  * initialized when the sub-database is first accessed. This function does
   7539  * the preliminary setup of the sub-cursor, filling in the fields that
   7540  * depend only on the parent DB.
   7541  * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized.
   7542  */
   7543 static void
   7544 mdb_xcursor_init0(MDB_cursor *mc)
   7545 {
   7546 	MDB_xcursor *mx = mc->mc_xcursor;
   7547 
   7548 	mx->mx_cursor.mc_xcursor = NULL;
   7549 	mx->mx_cursor.mc_txn = mc->mc_txn;
   7550 	mx->mx_cursor.mc_db = &mx->mx_db;
   7551 	mx->mx_cursor.mc_dbx = &mx->mx_dbx;
   7552 	mx->mx_cursor.mc_dbi = mc->mc_dbi;
   7553 	mx->mx_cursor.mc_dbflag = &mx->mx_dbflag;
   7554 	mx->mx_cursor.mc_snum = 0;
   7555 	mx->mx_cursor.mc_top = 0;
   7556 	mx->mx_cursor.mc_flags = C_SUB;
   7557 	mx->mx_dbx.md_name.mv_size = 0;
   7558 	mx->mx_dbx.md_name.mv_data = NULL;
   7559 	mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp;
   7560 	mx->mx_dbx.md_dcmp = NULL;
   7561 	mx->mx_dbx.md_rel = mc->mc_dbx->md_rel;
   7562 }
   7563 
   7564 /** Final setup of a sorted-dups cursor.
   7565  *	Sets up the fields that depend on the data from the main cursor.
   7566  * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized.
   7567  * @param[in] node The data containing the #MDB_db record for the
   7568  * sorted-dup database.
   7569  */
   7570 static void
   7571 mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
   7572 {
   7573 	MDB_xcursor *mx = mc->mc_xcursor;
   7574 
   7575 	if (node->mn_flags & F_SUBDATA) {
   7576 		memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db));
   7577 		mx->mx_cursor.mc_pg[0] = 0;
   7578 		mx->mx_cursor.mc_snum = 0;
   7579 		mx->mx_cursor.mc_top = 0;
   7580 		mx->mx_cursor.mc_flags = C_SUB;
   7581 	} else {
   7582 		MDB_page *fp = NODEDATA(node);
   7583 		mx->mx_db.md_pad = 0;
   7584 		mx->mx_db.md_flags = 0;
   7585 		mx->mx_db.md_depth = 1;
   7586 		mx->mx_db.md_branch_pages = 0;
   7587 		mx->mx_db.md_leaf_pages = 1;
   7588 		mx->mx_db.md_overflow_pages = 0;
   7589 		mx->mx_db.md_entries = NUMKEYS(fp);
   7590 		COPY_PGNO(mx->mx_db.md_root, MP_PGNO(fp));
   7591 		mx->mx_cursor.mc_snum = 1;
   7592 		mx->mx_cursor.mc_top = 0;
   7593 		mx->mx_cursor.mc_flags = C_INITIALIZED|C_SUB;
   7594 		mx->mx_cursor.mc_pg[0] = fp;
   7595 		mx->mx_cursor.mc_ki[0] = 0;
   7596 		if (mc->mc_db->md_flags & MDB_DUPFIXED) {
   7597 			mx->mx_db.md_flags = MDB_DUPFIXED;
   7598 			mx->mx_db.md_pad = fp->mp_pad;
   7599 			if (mc->mc_db->md_flags & MDB_INTEGERDUP)
   7600 				mx->mx_db.md_flags |= MDB_INTEGERKEY;
   7601 		}
   7602 	}
   7603 	DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi,
   7604 		mx->mx_db.md_root));
   7605 	mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA;
   7606 #if UINT_MAX < SIZE_MAX
   7607 	if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t))
   7608 		mx->mx_dbx.md_cmp = mdb_cmp_clong;
   7609 #endif
   7610 }
   7611 
   7612 
   7613 /** Fixup a sorted-dups cursor due to underlying update.
   7614  *	Sets up some fields that depend on the data from the main cursor.
   7615  *	Almost the same as init1, but skips initialization steps if the
   7616  *	xcursor had already been used.
   7617  * @param[in] mc The main cursor whose sorted-dups cursor is to be fixed up.
   7618  * @param[in] src_mx The xcursor of an up-to-date cursor.
   7619  * @param[in] new_dupdata True if converting from a non-#F_DUPDATA item.
   7620  */
   7621 static void
   7622 mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata)
   7623 {
   7624 	MDB_xcursor *mx = mc->mc_xcursor;
   7625 
   7626 	if (new_dupdata) {
   7627 		mx->mx_cursor.mc_snum = 1;
   7628 		mx->mx_cursor.mc_top = 0;
   7629 		mx->mx_cursor.mc_flags |= C_INITIALIZED;
   7630 		mx->mx_cursor.mc_ki[0] = 0;
   7631 		mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA;
   7632 #if UINT_MAX < SIZE_MAX
   7633 		mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp;
   7634 #endif
   7635 	} else if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) {
   7636 		return;
   7637 	}
   7638 	mx->mx_db = src_mx->mx_db;
   7639 	mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0];
   7640 	DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi,
   7641 		mx->mx_db.md_root));
   7642 }
   7643 
   7644 /** Initialize a cursor for a given transaction and database. */
   7645 static void
   7646 mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx)
   7647 {
   7648 	mc->mc_next = NULL;
   7649 	mc->mc_backup = NULL;
   7650 	mc->mc_dbi = dbi;
   7651 	mc->mc_txn = txn;
   7652 	mc->mc_db = &txn->mt_dbs[dbi];
   7653 	mc->mc_dbx = &txn->mt_dbxs[dbi];
   7654 	mc->mc_dbflag = &txn->mt_dbflags[dbi];
   7655 	mc->mc_snum = 0;
   7656 	mc->mc_top = 0;
   7657 	mc->mc_pg[0] = 0;
   7658 	mc->mc_ki[0] = 0;
   7659 	mc->mc_flags = 0;
   7660 	if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
   7661 		mdb_tassert(txn, mx != NULL);
   7662 		mc->mc_xcursor = mx;
   7663 		mdb_xcursor_init0(mc);
   7664 	} else {
   7665 		mc->mc_xcursor = NULL;
   7666 	}
   7667 	if (*mc->mc_dbflag & DB_STALE) {
   7668 		mdb_page_search(mc, NULL, MDB_PS_ROOTONLY);
   7669 	}
   7670 }
   7671 
   7672 int
   7673 mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret)
   7674 {
   7675 	MDB_cursor	*mc;
   7676 	size_t size = sizeof(MDB_cursor);
   7677 
   7678 	if (!ret || !TXN_DBI_EXIST(txn, dbi, DB_VALID))
   7679 		return EINVAL;
   7680 
   7681 	if (txn->mt_flags & MDB_TXN_BLOCKED)
   7682 		return MDB_BAD_TXN;
   7683 
   7684 	if (dbi == FREE_DBI && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY))
   7685 		return EINVAL;
   7686 
   7687 	if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT)
   7688 		size += sizeof(MDB_xcursor);
   7689 
   7690 	if ((mc = malloc(size)) != NULL) {
   7691 		mdb_cursor_init(mc, txn, dbi, (MDB_xcursor *)(mc + 1));
   7692 		if (txn->mt_cursors) {
   7693 			mc->mc_next = txn->mt_cursors[dbi];
   7694 			txn->mt_cursors[dbi] = mc;
   7695 			mc->mc_flags |= C_UNTRACK;
   7696 		}
   7697 	} else {
   7698 		return ENOMEM;
   7699 	}
   7700 
   7701 	*ret = mc;
   7702 
   7703 	return MDB_SUCCESS;
   7704 }
   7705 
   7706 int
   7707 mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc)
   7708 {
   7709 	if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi, DB_VALID))
   7710 		return EINVAL;
   7711 
   7712 	if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors)
   7713 		return EINVAL;
   7714 
   7715 	if (txn->mt_flags & MDB_TXN_BLOCKED)
   7716 		return MDB_BAD_TXN;
   7717 
   7718 	mdb_cursor_init(mc, txn, mc->mc_dbi, mc->mc_xcursor);
   7719 	return MDB_SUCCESS;
   7720 }
   7721 
   7722 /* Return the count of duplicate data items for the current key */
   7723 int
   7724 mdb_cursor_count(MDB_cursor *mc, size_t *countp)
   7725 {
   7726 	MDB_node	*leaf;
   7727 
   7728 	if (mc == NULL || countp == NULL)
   7729 		return EINVAL;
   7730 
   7731 	if (mc->mc_xcursor == NULL)
   7732 		return MDB_INCOMPATIBLE;
   7733 
   7734 	if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED)
   7735 		return MDB_BAD_TXN;
   7736 
   7737 	if (!(mc->mc_flags & C_INITIALIZED))
   7738 		return EINVAL;
   7739 
   7740 	if (!mc->mc_snum)
   7741 		return MDB_NOTFOUND;
   7742 
   7743 	if (mc->mc_flags & C_EOF) {
   7744 		if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top]))
   7745 			return MDB_NOTFOUND;
   7746 		mc->mc_flags ^= C_EOF;
   7747 	}
   7748 
   7749 	leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
   7750 	if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
   7751 		*countp = 1;
   7752 	} else {
   7753 		if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))
   7754 			return EINVAL;
   7755 
   7756 		*countp = mc->mc_xcursor->mx_db.md_entries;
   7757 	}
   7758 	return MDB_SUCCESS;
   7759 }
   7760 
   7761 void
   7762 mdb_cursor_close(MDB_cursor *mc)
   7763 {
   7764 	if (mc && !mc->mc_backup) {
   7765 		/* remove from txn, if tracked */
   7766 		if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) {
   7767 			MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi];
   7768 			while (*prev && *prev != mc) prev = &(*prev)->mc_next;
   7769 			if (*prev == mc)
   7770 				*prev = mc->mc_next;
   7771 		}
   7772 		free(mc);
   7773 	}
   7774 }
   7775 
   7776 MDB_txn *
   7777 mdb_cursor_txn(MDB_cursor *mc)
   7778 {
   7779 	if (!mc) return NULL;
   7780 	return mc->mc_txn;
   7781 }
   7782 
   7783 MDB_dbi
   7784 mdb_cursor_dbi(MDB_cursor *mc)
   7785 {
   7786 	return mc->mc_dbi;
   7787 }
   7788 
   7789 /** Replace the key for a branch node with a new key.
   7790  * Set #MDB_TXN_ERROR on failure.
   7791  * @param[in] mc Cursor pointing to the node to operate on.
   7792  * @param[in] key The new key to use.
   7793  * @return 0 on success, non-zero on failure.
   7794  */
   7795 static int
   7796 mdb_update_key(MDB_cursor *mc, MDB_val *key)
   7797 {
   7798 	MDB_page		*mp;
   7799 	MDB_node		*node;
   7800 	char			*base;
   7801 	size_t			 len;
   7802 	int				 delta, ksize, oksize;
   7803 	indx_t			 ptr, i, numkeys, indx;
   7804 	DKBUF;
   7805 
   7806 	indx = mc->mc_ki[mc->mc_top];
   7807 	mp = mc->mc_pg[mc->mc_top];
   7808 	node = NODEPTR(mp, indx);
   7809 	ptr = mp->mp_ptrs[indx];
   7810 #if MDB_DEBUG
   7811 	{
   7812 		MDB_val	k2;
   7813 		char kbuf2[DKBUF_MAXKEYSIZE*2+1];
   7814 		k2.mv_data = NODEKEY(node);
   7815 		k2.mv_size = node->mn_ksize;
   7816 		DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Z"u",
   7817 			indx, ptr,
   7818 			mdb_dkey(&k2, kbuf2),
   7819 			DKEY(key),
   7820 			mp->mp_pgno));
   7821 	}
   7822 #endif
   7823 
   7824 	/* Sizes must be 2-byte aligned. */
   7825 	ksize = EVEN(key->mv_size);
   7826 	oksize = EVEN(node->mn_ksize);
   7827 	delta = ksize - oksize;
   7828 
   7829 	/* Shift node contents if EVEN(key length) changed. */
   7830 	if (delta) {
   7831 		if (delta > 0 && SIZELEFT(mp) < delta) {
   7832 			pgno_t pgno;
   7833 			/* not enough space left, do a delete and split */
   7834 			DPRINTF(("Not enough room, delta = %d, splitting...", delta));
   7835 			pgno = NODEPGNO(node);
   7836 			mdb_node_del(mc, 0);
   7837 			return mdb_page_split(mc, key, NULL, pgno, MDB_SPLIT_REPLACE);
   7838 		}
   7839 
   7840 		numkeys = NUMKEYS(mp);
   7841 		for (i = 0; i < numkeys; i++) {
   7842 			if (mp->mp_ptrs[i] <= ptr)
   7843 				mp->mp_ptrs[i] -= delta;
   7844 		}
   7845 
   7846 		base = (char *)mp + mp->mp_upper + PAGEBASE;
   7847 		len = ptr - mp->mp_upper + NODESIZE;
   7848 		memmove(base - delta, base, len);
   7849 		mp->mp_upper -= delta;
   7850 
   7851 		node = NODEPTR(mp, indx);
   7852 	}
   7853 
   7854 	/* But even if no shift was needed, update ksize */
   7855 	if (node->mn_ksize != key->mv_size)
   7856 		node->mn_ksize = key->mv_size;
   7857 
   7858 	if (key->mv_size)
   7859 		memcpy(NODEKEY(node), key->mv_data, key->mv_size);
   7860 
   7861 	return MDB_SUCCESS;
   7862 }
   7863 
   7864 static void
   7865 mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst);
   7866 
   7867 /** Perform \b act while tracking temporary cursor \b mn */
   7868 #define WITH_CURSOR_TRACKING(mn, act) do { \
   7869 	MDB_cursor dummy, *tracked, **tp = &(mn).mc_txn->mt_cursors[mn.mc_dbi]; \
   7870 	if ((mn).mc_flags & C_SUB) { \
   7871 		dummy.mc_flags =  C_INITIALIZED; \
   7872 		dummy.mc_xcursor = (MDB_xcursor *)&(mn);	\
   7873 		tracked = &dummy; \
   7874 	} else { \
   7875 		tracked = &(mn); \
   7876 	} \
   7877 	tracked->mc_next = *tp; \
   7878 	*tp = tracked; \
   7879 	{ act; } \
   7880 	*tp = tracked->mc_next; \
   7881 } while (0)
   7882 
   7883 /** Move a node from csrc to cdst.
   7884  */
   7885 static int
   7886 mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
   7887 {
   7888 	MDB_node		*srcnode;
   7889 	MDB_val		 key, data;
   7890 	pgno_t	srcpg;
   7891 	MDB_cursor mn;
   7892 	int			 rc;
   7893 	unsigned short flags;
   7894 
   7895 	DKBUF;
   7896 
   7897 	/* Mark src and dst as dirty. */
   7898 	if ((rc = mdb_page_touch(csrc)) ||
   7899 	    (rc = mdb_page_touch(cdst)))
   7900 		return rc;
   7901 
   7902 	if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) {
   7903 		key.mv_size = csrc->mc_db->md_pad;
   7904 		key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size);
   7905 		data.mv_size = 0;
   7906 		data.mv_data = NULL;
   7907 		srcpg = 0;
   7908 		flags = 0;
   7909 	} else {
   7910 		srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top]);
   7911 		mdb_cassert(csrc, !((size_t)srcnode & 1));
   7912 		srcpg = NODEPGNO(srcnode);
   7913 		flags = srcnode->mn_flags;
   7914 		if (csrc->mc_ki[csrc->mc_top] == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) {
   7915 			unsigned int snum = csrc->mc_snum;
   7916 			MDB_node *s2;
   7917 			/* must find the lowest key below src */
   7918 			rc = mdb_page_search_lowest(csrc);
   7919 			if (rc)
   7920 				return rc;
   7921 			if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) {
   7922 				key.mv_size = csrc->mc_db->md_pad;
   7923 				key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size);
   7924 			} else {
   7925 				s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0);
   7926 				key.mv_size = NODEKSZ(s2);
   7927 				key.mv_data = NODEKEY(s2);
   7928 			}
   7929 			csrc->mc_snum = snum--;
   7930 			csrc->mc_top = snum;
   7931 		} else {
   7932 			key.mv_size = NODEKSZ(srcnode);
   7933 			key.mv_data = NODEKEY(srcnode);
   7934 		}
   7935 		data.mv_size = NODEDSZ(srcnode);
   7936 		data.mv_data = NODEDATA(srcnode);
   7937 	}
   7938 	mn.mc_xcursor = NULL;
   7939 	if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) {
   7940 		unsigned int snum = cdst->mc_snum;
   7941 		MDB_node *s2;
   7942 		MDB_val bkey;
   7943 		/* must find the lowest key below dst */
   7944 		mdb_cursor_copy(cdst, &mn);
   7945 		rc = mdb_page_search_lowest(&mn);
   7946 		if (rc)
   7947 			return rc;
   7948 		if (IS_LEAF2(mn.mc_pg[mn.mc_top])) {
   7949 			bkey.mv_size = mn.mc_db->md_pad;
   7950 			bkey.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, bkey.mv_size);
   7951 		} else {
   7952 			s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0);
   7953 			bkey.mv_size = NODEKSZ(s2);
   7954 			bkey.mv_data = NODEKEY(s2);
   7955 		}
   7956 		mn.mc_snum = snum--;
   7957 		mn.mc_top = snum;
   7958 		mn.mc_ki[snum] = 0;
   7959 		rc = mdb_update_key(&mn, &bkey);
   7960 		if (rc)
   7961 			return rc;
   7962 	}
   7963 
   7964 	DPRINTF(("moving %s node %u [%s] on page %"Z"u to node %u on page %"Z"u",
   7965 	    IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch",
   7966 	    csrc->mc_ki[csrc->mc_top],
   7967 		DKEY(&key),
   7968 	    csrc->mc_pg[csrc->mc_top]->mp_pgno,
   7969 	    cdst->mc_ki[cdst->mc_top], cdst->mc_pg[cdst->mc_top]->mp_pgno));
   7970 
   7971 	/* Add the node to the destination page.
   7972 	 */
   7973 	rc = mdb_node_add(cdst, cdst->mc_ki[cdst->mc_top], &key, &data, srcpg, flags);
   7974 	if (rc != MDB_SUCCESS)
   7975 		return rc;
   7976 
   7977 	/* Delete the node from the source page.
   7978 	 */
   7979 	mdb_node_del(csrc, key.mv_size);
   7980 
   7981 	{
   7982 		/* Adjust other cursors pointing to mp */
   7983 		MDB_cursor *m2, *m3;
   7984 		MDB_dbi dbi = csrc->mc_dbi;
   7985 		MDB_page *mpd, *mps;
   7986 
   7987 		mps = csrc->mc_pg[csrc->mc_top];
   7988 		/* If we're adding on the left, bump others up */
   7989 		if (fromleft) {
   7990 			mpd = cdst->mc_pg[csrc->mc_top];
   7991 			for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
   7992 				if (csrc->mc_flags & C_SUB)
   7993 					m3 = &m2->mc_xcursor->mx_cursor;
   7994 				else
   7995 					m3 = m2;
   7996 				if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top)
   7997 					continue;
   7998 				if (m3 != cdst &&
   7999 					m3->mc_pg[csrc->mc_top] == mpd &&
   8000 					m3->mc_ki[csrc->mc_top] >= cdst->mc_ki[csrc->mc_top]) {
   8001 					m3->mc_ki[csrc->mc_top]++;
   8002 				}
   8003 				if (m3 !=csrc &&
   8004 					m3->mc_pg[csrc->mc_top] == mps &&
   8005 					m3->mc_ki[csrc->mc_top] == csrc->mc_ki[csrc->mc_top]) {
   8006 					m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top];
   8007 					m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
   8008 					m3->mc_ki[csrc->mc_top-1]++;
   8009 				}
   8010 				if (IS_LEAF(mps))
   8011 					XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]);
   8012 			}
   8013 		} else
   8014 		/* Adding on the right, bump others down */
   8015 		{
   8016 			for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
   8017 				if (csrc->mc_flags & C_SUB)
   8018 					m3 = &m2->mc_xcursor->mx_cursor;
   8019 				else
   8020 					m3 = m2;
   8021 				if (m3 == csrc) continue;
   8022 				if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top)
   8023 					continue;
   8024 				if (m3->mc_pg[csrc->mc_top] == mps) {
   8025 					if (!m3->mc_ki[csrc->mc_top]) {
   8026 						m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top];
   8027 						m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
   8028 						m3->mc_ki[csrc->mc_top-1]--;
   8029 					} else {
   8030 						m3->mc_ki[csrc->mc_top]--;
   8031 					}
   8032 					if (IS_LEAF(mps))
   8033 						XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]);
   8034 				}
   8035 			}
   8036 		}
   8037 	}
   8038 
   8039 	/* Update the parent separators.
   8040 	 */
   8041 	if (csrc->mc_ki[csrc->mc_top] == 0) {
   8042 		if (csrc->mc_ki[csrc->mc_top-1] != 0) {
   8043 			if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) {
   8044 				key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size);
   8045 			} else {
   8046 				srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], 0);
   8047 				key.mv_size = NODEKSZ(srcnode);
   8048 				key.mv_data = NODEKEY(srcnode);
   8049 			}
   8050 			DPRINTF(("update separator for source page %"Z"u to [%s]",
   8051 				csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key)));
   8052 			mdb_cursor_copy(csrc, &mn);
   8053 			mn.mc_snum--;
   8054 			mn.mc_top--;
   8055 			/* We want mdb_rebalance to find mn when doing fixups */
   8056 			WITH_CURSOR_TRACKING(mn,
   8057 				rc = mdb_update_key(&mn, &key));
   8058 			if (rc)
   8059 				return rc;
   8060 		}
   8061 		if (IS_BRANCH(csrc->mc_pg[csrc->mc_top])) {
   8062 			MDB_val	 nullkey;
   8063 			indx_t	ix = csrc->mc_ki[csrc->mc_top];
   8064 			nullkey.mv_size = 0;
   8065 			csrc->mc_ki[csrc->mc_top] = 0;
   8066 			rc = mdb_update_key(csrc, &nullkey);
   8067 			csrc->mc_ki[csrc->mc_top] = ix;
   8068 			mdb_cassert(csrc, rc == MDB_SUCCESS);
   8069 		}
   8070 	}
   8071 
   8072 	if (cdst->mc_ki[cdst->mc_top] == 0) {
   8073 		if (cdst->mc_ki[cdst->mc_top-1] != 0) {
   8074 			if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) {
   8075 				key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size);
   8076 			} else {
   8077 				srcnode = NODEPTR(cdst->mc_pg[cdst->mc_top], 0);
   8078 				key.mv_size = NODEKSZ(srcnode);
   8079 				key.mv_data = NODEKEY(srcnode);
   8080 			}
   8081 			DPRINTF(("update separator for destination page %"Z"u to [%s]",
   8082 				cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key)));
   8083 			mdb_cursor_copy(cdst, &mn);
   8084 			mn.mc_snum--;
   8085 			mn.mc_top--;
   8086 			/* We want mdb_rebalance to find mn when doing fixups */
   8087 			WITH_CURSOR_TRACKING(mn,
   8088 				rc = mdb_update_key(&mn, &key));
   8089 			if (rc)
   8090 				return rc;
   8091 		}
   8092 		if (IS_BRANCH(cdst->mc_pg[cdst->mc_top])) {
   8093 			MDB_val	 nullkey;
   8094 			indx_t	ix = cdst->mc_ki[cdst->mc_top];
   8095 			nullkey.mv_size = 0;
   8096 			cdst->mc_ki[cdst->mc_top] = 0;
   8097 			rc = mdb_update_key(cdst, &nullkey);
   8098 			cdst->mc_ki[cdst->mc_top] = ix;
   8099 			mdb_cassert(cdst, rc == MDB_SUCCESS);
   8100 		}
   8101 	}
   8102 
   8103 	return MDB_SUCCESS;
   8104 }
   8105 
   8106 /** Merge one page into another.
   8107  *  The nodes from the page pointed to by \b csrc will
   8108  *	be copied to the page pointed to by \b cdst and then
   8109  *	the \b csrc page will be freed.
   8110  * @param[in] csrc Cursor pointing to the source page.
   8111  * @param[in] cdst Cursor pointing to the destination page.
   8112  * @return 0 on success, non-zero on failure.
   8113  */
   8114 static int
   8115 mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
   8116 {
   8117 	MDB_page	*psrc, *pdst;
   8118 	MDB_node	*srcnode;
   8119 	MDB_val		 key, data;
   8120 	unsigned	 nkeys;
   8121 	int			 rc;
   8122 	indx_t		 i, j;
   8123 
   8124 	psrc = csrc->mc_pg[csrc->mc_top];
   8125 	pdst = cdst->mc_pg[cdst->mc_top];
   8126 
   8127 	DPRINTF(("merging page %"Z"u into %"Z"u", psrc->mp_pgno, pdst->mp_pgno));
   8128 
   8129 	mdb_cassert(csrc, csrc->mc_snum > 1);	/* can't merge root page */
   8130 	mdb_cassert(csrc, cdst->mc_snum > 1);
   8131 
   8132 	/* Mark dst as dirty. */
   8133 	if ((rc = mdb_page_touch(cdst)))
   8134 		return rc;
   8135 
   8136 	/* get dst page again now that we've touched it. */
   8137 	pdst = cdst->mc_pg[cdst->mc_top];
   8138 
   8139 	/* Move all nodes from src to dst.
   8140 	 */
   8141 	j = nkeys = NUMKEYS(pdst);
   8142 	if (IS_LEAF2(psrc)) {
   8143 		key.mv_size = csrc->mc_db->md_pad;
   8144 		key.mv_data = METADATA(psrc);
   8145 		for (i = 0; i < NUMKEYS(psrc); i++, j++) {
   8146 			rc = mdb_node_add(cdst, j, &key, NULL, 0, 0);
   8147 			if (rc != MDB_SUCCESS)
   8148 				return rc;
   8149 			key.mv_data = (char *)key.mv_data + key.mv_size;
   8150 		}
   8151 	} else {
   8152 		for (i = 0; i < NUMKEYS(psrc); i++, j++) {
   8153 			srcnode = NODEPTR(psrc, i);
   8154 			if (i == 0 && IS_BRANCH(psrc)) {
   8155 				MDB_cursor mn;
   8156 				MDB_node *s2;
   8157 				mdb_cursor_copy(csrc, &mn);
   8158 				mn.mc_xcursor = NULL;
   8159 				/* must find the lowest key below src */
   8160 				rc = mdb_page_search_lowest(&mn);
   8161 				if (rc)
   8162 					return rc;
   8163 				if (IS_LEAF2(mn.mc_pg[mn.mc_top])) {
   8164 					key.mv_size = mn.mc_db->md_pad;
   8165 					key.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, key.mv_size);
   8166 				} else {
   8167 					s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0);
   8168 					key.mv_size = NODEKSZ(s2);
   8169 					key.mv_data = NODEKEY(s2);
   8170 				}
   8171 			} else {
   8172 				key.mv_size = srcnode->mn_ksize;
   8173 				key.mv_data = NODEKEY(srcnode);
   8174 			}
   8175 
   8176 			data.mv_size = NODEDSZ(srcnode);
   8177 			data.mv_data = NODEDATA(srcnode);
   8178 			rc = mdb_node_add(cdst, j, &key, &data, NODEPGNO(srcnode), srcnode->mn_flags);
   8179 			if (rc != MDB_SUCCESS)
   8180 				return rc;
   8181 		}
   8182 	}
   8183 
   8184 	DPRINTF(("dst page %"Z"u now has %u keys (%.1f%% filled)",
   8185 	    pdst->mp_pgno, NUMKEYS(pdst),
   8186 		(float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10));
   8187 
   8188 	/* Unlink the src page from parent and add to free list.
   8189 	 */
   8190 	csrc->mc_top--;
   8191 	mdb_node_del(csrc, 0);
   8192 	if (csrc->mc_ki[csrc->mc_top] == 0) {
   8193 		key.mv_size = 0;
   8194 		rc = mdb_update_key(csrc, &key);
   8195 		if (rc) {
   8196 			csrc->mc_top++;
   8197 			return rc;
   8198 		}
   8199 	}
   8200 	csrc->mc_top++;
   8201 
   8202 	psrc = csrc->mc_pg[csrc->mc_top];
   8203 	/* If not operating on FreeDB, allow this page to be reused
   8204 	 * in this txn. Otherwise just add to free list.
   8205 	 */
   8206 	rc = mdb_page_loose(csrc, psrc);
   8207 	if (rc)
   8208 		return rc;
   8209 	if (IS_LEAF(psrc))
   8210 		csrc->mc_db->md_leaf_pages--;
   8211 	else
   8212 		csrc->mc_db->md_branch_pages--;
   8213 	{
   8214 		/* Adjust other cursors pointing to mp */
   8215 		MDB_cursor *m2, *m3;
   8216 		MDB_dbi dbi = csrc->mc_dbi;
   8217 		unsigned int top = csrc->mc_top;
   8218 
   8219 		for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
   8220 			if (csrc->mc_flags & C_SUB)
   8221 				m3 = &m2->mc_xcursor->mx_cursor;
   8222 			else
   8223 				m3 = m2;
   8224 			if (m3 == csrc) continue;
   8225 			if (m3->mc_snum < csrc->mc_snum) continue;
   8226 			if (m3->mc_pg[top] == psrc) {
   8227 				m3->mc_pg[top] = pdst;
   8228 				m3->mc_ki[top] += nkeys;
   8229 				m3->mc_ki[top-1] = cdst->mc_ki[top-1];
   8230 			} else if (m3->mc_pg[top-1] == csrc->mc_pg[top-1] &&
   8231 				m3->mc_ki[top-1] > csrc->mc_ki[top-1]) {
   8232 				m3->mc_ki[top-1]--;
   8233 			}
   8234 			if (IS_LEAF(psrc))
   8235 				XCURSOR_REFRESH(m3, top, m3->mc_pg[top]);
   8236 		}
   8237 	}
   8238 	{
   8239 		unsigned int snum = cdst->mc_snum;
   8240 		uint16_t depth = cdst->mc_db->md_depth;
   8241 		mdb_cursor_pop(cdst);
   8242 		rc = mdb_rebalance(cdst);
   8243 		/* Did the tree height change? */
   8244 		if (depth != cdst->mc_db->md_depth)
   8245 			snum += cdst->mc_db->md_depth - depth;
   8246 		cdst->mc_snum = snum;
   8247 		cdst->mc_top = snum-1;
   8248 	}
   8249 	return rc;
   8250 }
   8251 
   8252 /** Copy the contents of a cursor.
   8253  * @param[in] csrc The cursor to copy from.
   8254  * @param[out] cdst The cursor to copy to.
   8255  */
   8256 static void
   8257 mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst)
   8258 {
   8259 	unsigned int i;
   8260 
   8261 	cdst->mc_txn = csrc->mc_txn;
   8262 	cdst->mc_dbi = csrc->mc_dbi;
   8263 	cdst->mc_db  = csrc->mc_db;
   8264 	cdst->mc_dbx = csrc->mc_dbx;
   8265 	cdst->mc_snum = csrc->mc_snum;
   8266 	cdst->mc_top = csrc->mc_top;
   8267 	cdst->mc_flags = csrc->mc_flags;
   8268 
   8269 	for (i=0; i<csrc->mc_snum; i++) {
   8270 		cdst->mc_pg[i] = csrc->mc_pg[i];
   8271 		cdst->mc_ki[i] = csrc->mc_ki[i];
   8272 	}
   8273 }
   8274 
   8275 /** Rebalance the tree after a delete operation.
   8276  * @param[in] mc Cursor pointing to the page where rebalancing
   8277  * should begin.
   8278  * @return 0 on success, non-zero on failure.
   8279  */
   8280 static int
   8281 mdb_rebalance(MDB_cursor *mc)
   8282 {
   8283 	MDB_node	*node;
   8284 	int rc, fromleft;
   8285 	unsigned int ptop, minkeys, thresh;
   8286 	MDB_cursor	mn;
   8287 	indx_t oldki;
   8288 
   8289 	if (IS_BRANCH(mc->mc_pg[mc->mc_top])) {
   8290 		minkeys = 2;
   8291 		thresh = 1;
   8292 	} else {
   8293 		minkeys = 1;
   8294 		thresh = FILL_THRESHOLD;
   8295 	}
   8296 	DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)",
   8297 	    IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch",
   8298 	    mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]),
   8299 		(float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10));
   8300 
   8301 	if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= thresh &&
   8302 		NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) {
   8303 		DPRINTF(("no need to rebalance page %"Z"u, above fill threshold",
   8304 		    mdb_dbg_pgno(mc->mc_pg[mc->mc_top])));
   8305 		return MDB_SUCCESS;
   8306 	}
   8307 
   8308 	if (mc->mc_snum < 2) {
   8309 		MDB_page *mp = mc->mc_pg[0];
   8310 		if (IS_SUBP(mp)) {
   8311 			DPUTS("Can't rebalance a subpage, ignoring");
   8312 			return MDB_SUCCESS;
   8313 		}
   8314 		if (NUMKEYS(mp) == 0) {
   8315 			DPUTS("tree is completely empty");
   8316 			mc->mc_db->md_root = P_INVALID;
   8317 			mc->mc_db->md_depth = 0;
   8318 			mc->mc_db->md_leaf_pages = 0;
   8319 			rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno);
   8320 			if (rc)
   8321 				return rc;
   8322 			/* Adjust cursors pointing to mp */
   8323 			mc->mc_snum = 0;
   8324 			mc->mc_top = 0;
   8325 			mc->mc_flags &= ~C_INITIALIZED;
   8326 			{
   8327 				MDB_cursor *m2, *m3;
   8328 				MDB_dbi dbi = mc->mc_dbi;
   8329 
   8330 				for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
   8331 					if (mc->mc_flags & C_SUB)
   8332 						m3 = &m2->mc_xcursor->mx_cursor;
   8333 					else
   8334 						m3 = m2;
   8335 					if (!(m3->mc_flags & C_INITIALIZED) || (m3->mc_snum < mc->mc_snum))
   8336 						continue;
   8337 					if (m3->mc_pg[0] == mp) {
   8338 						m3->mc_snum = 0;
   8339 						m3->mc_top = 0;
   8340 						m3->mc_flags &= ~C_INITIALIZED;
   8341 					}
   8342 				}
   8343 			}
   8344 		} else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) {
   8345 			int i;
   8346 			DPUTS("collapsing root page!");
   8347 			rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno);
   8348 			if (rc)
   8349 				return rc;
   8350 			mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0));
   8351 			rc = mdb_page_get(mc, mc->mc_db->md_root, &mc->mc_pg[0], NULL);
   8352 			if (rc)
   8353 				return rc;
   8354 			mc->mc_db->md_depth--;
   8355 			mc->mc_db->md_branch_pages--;
   8356 			mc->mc_ki[0] = mc->mc_ki[1];
   8357 			for (i = 1; i<mc->mc_db->md_depth; i++) {
   8358 				mc->mc_pg[i] = mc->mc_pg[i+1];
   8359 				mc->mc_ki[i] = mc->mc_ki[i+1];
   8360 			}
   8361 			{
   8362 				/* Adjust other cursors pointing to mp */
   8363 				MDB_cursor *m2, *m3;
   8364 				MDB_dbi dbi = mc->mc_dbi;
   8365 
   8366 				for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
   8367 					if (mc->mc_flags & C_SUB)
   8368 						m3 = &m2->mc_xcursor->mx_cursor;
   8369 					else
   8370 						m3 = m2;
   8371 					if (m3 == mc) continue;
   8372 					if (!(m3->mc_flags & C_INITIALIZED))
   8373 						continue;
   8374 					if (m3->mc_pg[0] == mp) {
   8375 						for (i=0; i<mc->mc_db->md_depth; i++) {
   8376 							m3->mc_pg[i] = m3->mc_pg[i+1];
   8377 							m3->mc_ki[i] = m3->mc_ki[i+1];
   8378 						}
   8379 						m3->mc_snum--;
   8380 						m3->mc_top--;
   8381 					}
   8382 				}
   8383 			}
   8384 		} else
   8385 			DPUTS("root page doesn't need rebalancing");
   8386 		return MDB_SUCCESS;
   8387 	}
   8388 
   8389 	/* The parent (branch page) must have at least 2 pointers,
   8390 	 * otherwise the tree is invalid.
   8391 	 */
   8392 	ptop = mc->mc_top-1;
   8393 	mdb_cassert(mc, NUMKEYS(mc->mc_pg[ptop]) > 1);
   8394 
   8395 	/* Leaf page fill factor is below the threshold.
   8396 	 * Try to move keys from left or right neighbor, or
   8397 	 * merge with a neighbor page.
   8398 	 */
   8399 
   8400 	/* Find neighbors.
   8401 	 */
   8402 	mdb_cursor_copy(mc, &mn);
   8403 	mn.mc_xcursor = NULL;
   8404 
   8405 	oldki = mc->mc_ki[mc->mc_top];
   8406 	if (mc->mc_ki[ptop] == 0) {
   8407 		/* We're the leftmost leaf in our parent.
   8408 		 */
   8409 		DPUTS("reading right neighbor");
   8410 		mn.mc_ki[ptop]++;
   8411 		node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]);
   8412 		rc = mdb_page_get(mc, NODEPGNO(node), &mn.mc_pg[mn.mc_top], NULL);
   8413 		if (rc)
   8414 			return rc;
   8415 		mn.mc_ki[mn.mc_top] = 0;
   8416 		mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]);
   8417 		fromleft = 0;
   8418 	} else {
   8419 		/* There is at least one neighbor to the left.
   8420 		 */
   8421 		DPUTS("reading left neighbor");
   8422 		mn.mc_ki[ptop]--;
   8423 		node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]);
   8424 		rc = mdb_page_get(mc, NODEPGNO(node), &mn.mc_pg[mn.mc_top], NULL);
   8425 		if (rc)
   8426 			return rc;
   8427 		mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1;
   8428 		mc->mc_ki[mc->mc_top] = 0;
   8429 		fromleft = 1;
   8430 	}
   8431 
   8432 	DPRINTF(("found neighbor page %"Z"u (%u keys, %.1f%% full)",
   8433 	    mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]),
   8434 		(float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10));
   8435 
   8436 	/* If the neighbor page is above threshold and has enough keys,
   8437 	 * move one key from it. Otherwise we should try to merge them.
   8438 	 * (A branch page must never have less than 2 keys.)
   8439 	 */
   8440 	if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= thresh && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) {
   8441 		rc = mdb_node_move(&mn, mc, fromleft);
   8442 		if (fromleft) {
   8443 			/* if we inserted on left, bump position up */
   8444 			oldki++;
   8445 		}
   8446 	} else {
   8447 		if (!fromleft) {
   8448 			rc = mdb_page_merge(&mn, mc);
   8449 		} else {
   8450 			oldki += NUMKEYS(mn.mc_pg[mn.mc_top]);
   8451 			mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
   8452 			/* We want mdb_rebalance to find mn when doing fixups */
   8453 			WITH_CURSOR_TRACKING(mn,
   8454 				rc = mdb_page_merge(mc, &mn));
   8455 			mdb_cursor_copy(&mn, mc);
   8456 		}
   8457 		mc->mc_flags &= ~C_EOF;
   8458 	}
   8459 	mc->mc_ki[mc->mc_top] = oldki;
   8460 	return rc;
   8461 }
   8462 
   8463 /** Complete a delete operation started by #mdb_cursor_del(). */
   8464 static int
   8465 mdb_cursor_del0(MDB_cursor *mc)
   8466 {
   8467 	int rc;
   8468 	MDB_page *mp;
   8469 	indx_t ki;
   8470 	unsigned int nkeys;
   8471 	MDB_cursor *m2, *m3;
   8472 	MDB_dbi dbi = mc->mc_dbi;
   8473 
   8474 	ki = mc->mc_ki[mc->mc_top];
   8475 	mp = mc->mc_pg[mc->mc_top];
   8476 	mdb_node_del(mc, mc->mc_db->md_pad);
   8477 	mc->mc_db->md_entries--;
   8478 	{
   8479 		/* Adjust other cursors pointing to mp */
   8480 		for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
   8481 			m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
   8482 			if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED))
   8483 				continue;
   8484 			if (m3 == mc || m3->mc_snum < mc->mc_snum)
   8485 				continue;
   8486 			if (m3->mc_pg[mc->mc_top] == mp) {
   8487 				if (m3->mc_ki[mc->mc_top] == ki) {
   8488 					m3->mc_flags |= C_DEL;
   8489 					if (mc->mc_db->md_flags & MDB_DUPSORT) {
   8490 						/* Sub-cursor referred into dataset which is gone */
   8491 						m3->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
   8492 					}
   8493 					continue;
   8494 				} else if (m3->mc_ki[mc->mc_top] > ki) {
   8495 					m3->mc_ki[mc->mc_top]--;
   8496 				}
   8497 				XCURSOR_REFRESH(m3, mc->mc_top, mp);
   8498 			}
   8499 		}
   8500 	}
   8501 	rc = mdb_rebalance(mc);
   8502 	if (rc)
   8503 		goto fail;
   8504 
   8505 	/* DB is totally empty now, just bail out.
   8506 	 * Other cursors adjustments were already done
   8507 	 * by mdb_rebalance and aren't needed here.
   8508 	 */
   8509 	if (!mc->mc_snum) {
   8510 		mc->mc_flags |= C_EOF;
   8511 		return rc;
   8512 	}
   8513 
   8514 	mp = mc->mc_pg[mc->mc_top];
   8515 	nkeys = NUMKEYS(mp);
   8516 
   8517 	/* Adjust other cursors pointing to mp */
   8518 	for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) {
   8519 		m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
   8520 		if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED))
   8521 			continue;
   8522 		if (m3->mc_snum < mc->mc_snum)
   8523 			continue;
   8524 		if (m3->mc_pg[mc->mc_top] == mp) {
   8525 			if (m3->mc_ki[mc->mc_top] >= mc->mc_ki[mc->mc_top]) {
   8526 			/* if m3 points past last node in page, find next sibling */
   8527 				if (m3->mc_ki[mc->mc_top] >= nkeys) {
   8528 					rc = mdb_cursor_sibling(m3, 1);
   8529 					if (rc == MDB_NOTFOUND) {
   8530 						m3->mc_flags |= C_EOF;
   8531 						rc = MDB_SUCCESS;
   8532 						continue;
   8533 					}
   8534 					if (rc)
   8535 						goto fail;
   8536 				}
   8537 				if (m3->mc_xcursor && !(m3->mc_flags & C_EOF)) {
   8538 					MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
   8539 					/* If this node has dupdata, it may need to be reinited
   8540 					 * because its data has moved.
   8541 					 * If the xcursor was not initd it must be reinited.
   8542 					 * Else if node points to a subDB, nothing is needed.
   8543 					 * Else (xcursor was initd, not a subDB) needs mc_pg[0] reset.
   8544 					 */
   8545 					if (node->mn_flags & F_DUPDATA) {
   8546 						if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
   8547 							if (!(node->mn_flags & F_SUBDATA))
   8548 								m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
   8549 						} else {
   8550 							mdb_xcursor_init1(m3, node);
   8551 							rc = mdb_cursor_first(&m3->mc_xcursor->mx_cursor, NULL, NULL);
   8552 							if (rc)
   8553 								goto fail;
   8554 						}
   8555 					}
   8556 					m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL;
   8557 				}
   8558 			}
   8559 		}
   8560 	}
   8561 	mc->mc_flags |= C_DEL;
   8562 
   8563 fail:
   8564 	if (rc)
   8565 		mc->mc_txn->mt_flags |= MDB_TXN_ERROR;
   8566 	return rc;
   8567 }
   8568 
   8569 int
   8570 mdb_del(MDB_txn *txn, MDB_dbi dbi,
   8571     MDB_val *key, MDB_val *data)
   8572 {
   8573 	if (!key || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
   8574 		return EINVAL;
   8575 
   8576 	if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
   8577 		return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
   8578 
   8579 	if (!F_ISSET(txn->mt_dbs[dbi].md_flags, MDB_DUPSORT)) {
   8580 		/* must ignore any data */
   8581 		data = NULL;
   8582 	}
   8583 
   8584 	return mdb_del0(txn, dbi, key, data, 0);
   8585 }
   8586 
   8587 static int
   8588 mdb_del0(MDB_txn *txn, MDB_dbi dbi,
   8589 	MDB_val *key, MDB_val *data, unsigned flags)
   8590 {
   8591 	MDB_cursor mc;
   8592 	MDB_xcursor mx;
   8593 	MDB_cursor_op op;
   8594 	MDB_val rdata, *xdata;
   8595 	int		 rc, exact = 0;
   8596 	DKBUF;
   8597 
   8598 	DPRINTF(("====> delete db %u key [%s]", dbi, DKEY(key)));
   8599 
   8600 	mdb_cursor_init(&mc, txn, dbi, &mx);
   8601 
   8602 	if (data) {
   8603 		op = MDB_GET_BOTH;
   8604 		rdata = *data;
   8605 		xdata = &rdata;
   8606 	} else {
   8607 		op = MDB_SET;
   8608 		xdata = NULL;
   8609 		flags |= MDB_NODUPDATA;
   8610 	}
   8611 	rc = mdb_cursor_set(&mc, key, xdata, op, &exact);
   8612 	if (rc == 0) {
   8613 		/* let mdb_page_split know about this cursor if needed:
   8614 		 * delete will trigger a rebalance; if it needs to move
   8615 		 * a node from one page to another, it will have to
   8616 		 * update the parent's separator key(s). If the new sepkey
   8617 		 * is larger than the current one, the parent page may
   8618 		 * run out of space, triggering a split. We need this
   8619 		 * cursor to be consistent until the end of the rebalance.
   8620 		 */
   8621 		mc.mc_flags |= C_UNTRACK;
   8622 		mc.mc_next = txn->mt_cursors[dbi];
   8623 		txn->mt_cursors[dbi] = &mc;
   8624 		rc = mdb_cursor_del(&mc, flags);
   8625 		txn->mt_cursors[dbi] = mc.mc_next;
   8626 	}
   8627 	return rc;
   8628 }
   8629 
   8630 /** Split a page and insert a new node.
   8631  * Set #MDB_TXN_ERROR on failure.
   8632  * @param[in,out] mc Cursor pointing to the page and desired insertion index.
   8633  * The cursor will be updated to point to the actual page and index where
   8634  * the node got inserted after the split.
   8635  * @param[in] newkey The key for the newly inserted node.
   8636  * @param[in] newdata The data for the newly inserted node.
   8637  * @param[in] newpgno The page number, if the new node is a branch node.
   8638  * @param[in] nflags The #NODE_ADD_FLAGS for the new node.
   8639  * @return 0 on success, non-zero on failure.
   8640  */
   8641 static int
   8642 mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno,
   8643 	unsigned int nflags)
   8644 {
   8645 	unsigned int flags;
   8646 	int		 rc = MDB_SUCCESS, new_root = 0, did_split = 0;
   8647 	indx_t		 newindx;
   8648 	pgno_t		 pgno = 0;
   8649 	int	 i, j, split_indx, nkeys, pmax;
   8650 	MDB_env 	*env = mc->mc_txn->mt_env;
   8651 	MDB_node	*node;
   8652 	MDB_val	 sepkey, rkey, xdata, *rdata = &xdata;
   8653 	MDB_page	*copy = NULL;
   8654 	MDB_page	*mp, *rp, *pp;
   8655 	int ptop;
   8656 	MDB_cursor	mn;
   8657 	DKBUF;
   8658 
   8659 	mp = mc->mc_pg[mc->mc_top];
   8660 	newindx = mc->mc_ki[mc->mc_top];
   8661 	nkeys = NUMKEYS(mp);
   8662 
   8663 	DPRINTF(("-----> splitting %s page %"Z"u and adding [%s] at index %i/%i",
   8664 	    IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno,
   8665 	    DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys));
   8666 
   8667 	/* Create a right sibling. */
   8668 	if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp)))
   8669 		return rc;
   8670 	rp->mp_pad = mp->mp_pad;
   8671 	DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno));
   8672 
   8673 	/* Usually when splitting the root page, the cursor
   8674 	 * height is 1. But when called from mdb_update_key,
   8675 	 * the cursor height may be greater because it walks
   8676 	 * up the stack while finding the branch slot to update.
   8677 	 */
   8678 	if (mc->mc_top < 1) {
   8679 		if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp)))
   8680 			goto done;
   8681 		/* shift current top to make room for new parent */
   8682 		for (i=mc->mc_snum; i>0; i--) {
   8683 			mc->mc_pg[i] = mc->mc_pg[i-1];
   8684 			mc->mc_ki[i] = mc->mc_ki[i-1];
   8685 		}
   8686 		mc->mc_pg[0] = pp;
   8687 		mc->mc_ki[0] = 0;
   8688 		mc->mc_db->md_root = pp->mp_pgno;
   8689 		DPRINTF(("root split! new root = %"Z"u", pp->mp_pgno));
   8690 		new_root = mc->mc_db->md_depth++;
   8691 
   8692 		/* Add left (implicit) pointer. */
   8693 		if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) {
   8694 			/* undo the pre-push */
   8695 			mc->mc_pg[0] = mc->mc_pg[1];
   8696 			mc->mc_ki[0] = mc->mc_ki[1];
   8697 			mc->mc_db->md_root = mp->mp_pgno;
   8698 			mc->mc_db->md_depth--;
   8699 			goto done;
   8700 		}
   8701 		mc->mc_snum++;
   8702 		mc->mc_top++;
   8703 		ptop = 0;
   8704 	} else {
   8705 		ptop = mc->mc_top-1;
   8706 		DPRINTF(("parent branch page is %"Z"u", mc->mc_pg[ptop]->mp_pgno));
   8707 	}
   8708 
   8709 	mdb_cursor_copy(mc, &mn);
   8710 	mn.mc_xcursor = NULL;
   8711 	mn.mc_pg[mn.mc_top] = rp;
   8712 	mn.mc_ki[ptop] = mc->mc_ki[ptop]+1;
   8713 
   8714 	if (nflags & MDB_APPEND) {
   8715 		mn.mc_ki[mn.mc_top] = 0;
   8716 		sepkey = *newkey;
   8717 		split_indx = newindx;
   8718 		nkeys = 0;
   8719 	} else {
   8720 
   8721 		split_indx = (nkeys+1) / 2;
   8722 
   8723 		if (IS_LEAF2(rp)) {
   8724 			char *split, *ins;
   8725 			int x;
   8726 			unsigned int lsize, rsize, ksize;
   8727 			/* Move half of the keys to the right sibling */
   8728 			x = mc->mc_ki[mc->mc_top] - split_indx;
   8729 			ksize = mc->mc_db->md_pad;
   8730 			split = LEAF2KEY(mp, split_indx, ksize);
   8731 			rsize = (nkeys - split_indx) * ksize;
   8732 			lsize = (nkeys - split_indx) * sizeof(indx_t);
   8733 			mp->mp_lower -= lsize;
   8734 			rp->mp_lower += lsize;
   8735 			mp->mp_upper += rsize - lsize;
   8736 			rp->mp_upper -= rsize - lsize;
   8737 			sepkey.mv_size = ksize;
   8738 			if (newindx == split_indx) {
   8739 				sepkey.mv_data = newkey->mv_data;
   8740 			} else {
   8741 				sepkey.mv_data = split;
   8742 			}
   8743 			if (x<0) {
   8744 				ins = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], ksize);
   8745 				memcpy(rp->mp_ptrs, split, rsize);
   8746 				sepkey.mv_data = rp->mp_ptrs;
   8747 				memmove(ins+ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize);
   8748 				memcpy(ins, newkey->mv_data, ksize);
   8749 				mp->mp_lower += sizeof(indx_t);
   8750 				mp->mp_upper -= ksize - sizeof(indx_t);
   8751 			} else {
   8752 				if (x)
   8753 					memcpy(rp->mp_ptrs, split, x * ksize);
   8754 				ins = LEAF2KEY(rp, x, ksize);
   8755 				memcpy(ins, newkey->mv_data, ksize);
   8756 				memcpy(ins+ksize, split + x * ksize, rsize - x * ksize);
   8757 				rp->mp_lower += sizeof(indx_t);
   8758 				rp->mp_upper -= ksize - sizeof(indx_t);
   8759 				mc->mc_ki[mc->mc_top] = x;
   8760 			}
   8761 		} else {
   8762 			int psize, nsize, k, keythresh;
   8763 
   8764 			/* Maximum free space in an empty page */
   8765 			pmax = env->me_psize - PAGEHDRSZ;
   8766 			/* Threshold number of keys considered "small" */
   8767 			keythresh = env->me_psize >> 7;
   8768 
   8769 			if (IS_LEAF(mp))
   8770 				nsize = mdb_leaf_size(env, newkey, newdata);
   8771 			else
   8772 				nsize = mdb_branch_size(env, newkey);
   8773 			nsize = EVEN(nsize);
   8774 
   8775 			/* grab a page to hold a temporary copy */
   8776 			copy = mdb_page_malloc(mc->mc_txn, 1);
   8777 			if (copy == NULL) {
   8778 				rc = ENOMEM;
   8779 				goto done;
   8780 			}
   8781 			copy->mp_pgno  = mp->mp_pgno;
   8782 			copy->mp_flags = mp->mp_flags;
   8783 			copy->mp_lower = (PAGEHDRSZ-PAGEBASE);
   8784 			copy->mp_upper = env->me_psize - PAGEBASE;
   8785 
   8786 			/* prepare to insert */
   8787 			for (i=0, j=0; i<nkeys; i++) {
   8788 				if (i == newindx) {
   8789 					copy->mp_ptrs[j++] = 0;
   8790 				}
   8791 				copy->mp_ptrs[j++] = mp->mp_ptrs[i];
   8792 			}
   8793 
   8794 			/* When items are relatively large the split point needs
   8795 			 * to be checked, because being off-by-one will make the
   8796 			 * difference between success or failure in mdb_node_add.
   8797 			 *
   8798 			 * It's also relevant if a page happens to be laid out
   8799 			 * such that one half of its nodes are all "small" and
   8800 			 * the other half of its nodes are "large." If the new
   8801 			 * item is also "large" and falls on the half with
   8802 			 * "large" nodes, it also may not fit.
   8803 			 *
   8804 			 * As a final tweak, if the new item goes on the last
   8805 			 * spot on the page (and thus, onto the new page), bias
   8806 			 * the split so the new page is emptier than the old page.
   8807 			 * This yields better packing during sequential inserts.
   8808 			 */
   8809 			if (nkeys < keythresh || nsize > pmax/16 || newindx >= nkeys) {
   8810 				/* Find split point */
   8811 				psize = 0;
   8812 				if (newindx <= split_indx || newindx >= nkeys) {
   8813 					i = 0; j = 1;
   8814 					k = newindx >= nkeys ? nkeys : split_indx+1+IS_LEAF(mp);
   8815 				} else {
   8816 					i = nkeys; j = -1;
   8817 					k = split_indx-1;
   8818 				}
   8819 				for (; i!=k; i+=j) {
   8820 					if (i == newindx) {
   8821 						psize += nsize;
   8822 						node = NULL;
   8823 					} else {
   8824 						node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE);
   8825 						psize += NODESIZE + NODEKSZ(node) + sizeof(indx_t);
   8826 						if (IS_LEAF(mp)) {
   8827 							if (F_ISSET(node->mn_flags, F_BIGDATA))
   8828 								psize += sizeof(pgno_t);
   8829 							else
   8830 								psize += NODEDSZ(node);
   8831 						}
   8832 						psize = EVEN(psize);
   8833 					}
   8834 					if (psize > pmax || i == k-j) {
   8835 						split_indx = i + (j<0);
   8836 						break;
   8837 					}
   8838 				}
   8839 			}
   8840 			if (split_indx == newindx) {
   8841 				sepkey.mv_size = newkey->mv_size;
   8842 				sepkey.mv_data = newkey->mv_data;
   8843 			} else {
   8844 				node = (MDB_node *)((char *)mp + copy->mp_ptrs[split_indx] + PAGEBASE);
   8845 				sepkey.mv_size = node->mn_ksize;
   8846 				sepkey.mv_data = NODEKEY(node);
   8847 			}
   8848 		}
   8849 	}
   8850 
   8851 	DPRINTF(("separator is %d [%s]", split_indx, DKEY(&sepkey)));
   8852 
   8853 	/* Copy separator key to the parent.
   8854 	 */
   8855 	if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) {
   8856 		int snum = mc->mc_snum;
   8857 		mn.mc_snum--;
   8858 		mn.mc_top--;
   8859 		did_split = 1;
   8860 		/* We want other splits to find mn when doing fixups */
   8861 		WITH_CURSOR_TRACKING(mn,
   8862 			rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0));
   8863 		if (rc)
   8864 			goto done;
   8865 
   8866 		/* root split? */
   8867 		if (mc->mc_snum > snum) {
   8868 			ptop++;
   8869 		}
   8870 		/* Right page might now have changed parent.
   8871 		 * Check if left page also changed parent.
   8872 		 */
   8873 		if (mn.mc_pg[ptop] != mc->mc_pg[ptop] &&
   8874 		    mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) {
   8875 			for (i=0; i<ptop; i++) {
   8876 				mc->mc_pg[i] = mn.mc_pg[i];
   8877 				mc->mc_ki[i] = mn.mc_ki[i];
   8878 			}
   8879 			mc->mc_pg[ptop] = mn.mc_pg[ptop];
   8880 			if (mn.mc_ki[ptop]) {
   8881 				mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1;
   8882 			} else {
   8883 				/* find right page's left sibling */
   8884 				mc->mc_ki[ptop] = mn.mc_ki[ptop];
   8885 				mdb_cursor_sibling(mc, 0);
   8886 			}
   8887 		}
   8888 	} else {
   8889 		mn.mc_top--;
   8890 		rc = mdb_node_add(&mn, mn.mc_ki[ptop], &sepkey, NULL, rp->mp_pgno, 0);
   8891 		mn.mc_top++;
   8892 	}
   8893 	if (rc != MDB_SUCCESS) {
   8894 		goto done;
   8895 	}
   8896 	if (nflags & MDB_APPEND) {
   8897 		mc->mc_pg[mc->mc_top] = rp;
   8898 		mc->mc_ki[mc->mc_top] = 0;
   8899 		rc = mdb_node_add(mc, 0, newkey, newdata, newpgno, nflags);
   8900 		if (rc)
   8901 			goto done;
   8902 		for (i=0; i<mc->mc_top; i++)
   8903 			mc->mc_ki[i] = mn.mc_ki[i];
   8904 	} else if (!IS_LEAF2(mp)) {
   8905 		/* Move nodes */
   8906 		mc->mc_pg[mc->mc_top] = rp;
   8907 		i = split_indx;
   8908 		j = 0;
   8909 		do {
   8910 			if (i == newindx) {
   8911 				rkey.mv_data = newkey->mv_data;
   8912 				rkey.mv_size = newkey->mv_size;
   8913 				if (IS_LEAF(mp)) {
   8914 					rdata = newdata;
   8915 				} else
   8916 					pgno = newpgno;
   8917 				flags = nflags;
   8918 				/* Update index for the new key. */
   8919 				mc->mc_ki[mc->mc_top] = j;
   8920 			} else {
   8921 				node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE);
   8922 				rkey.mv_data = NODEKEY(node);
   8923 				rkey.mv_size = node->mn_ksize;
   8924 				if (IS_LEAF(mp)) {
   8925 					xdata.mv_data = NODEDATA(node);
   8926 					xdata.mv_size = NODEDSZ(node);
   8927 					rdata = &xdata;
   8928 				} else
   8929 					pgno = NODEPGNO(node);
   8930 				flags = node->mn_flags;
   8931 			}
   8932 
   8933 			if (!IS_LEAF(mp) && j == 0) {
   8934 				/* First branch index doesn't need key data. */
   8935 				rkey.mv_size = 0;
   8936 			}
   8937 
   8938 			rc = mdb_node_add(mc, j, &rkey, rdata, pgno, flags);
   8939 			if (rc)
   8940 				goto done;
   8941 			if (i == nkeys) {
   8942 				i = 0;
   8943 				j = 0;
   8944 				mc->mc_pg[mc->mc_top] = copy;
   8945 			} else {
   8946 				i++;
   8947 				j++;
   8948 			}
   8949 		} while (i != split_indx);
   8950 
   8951 		nkeys = NUMKEYS(copy);
   8952 		for (i=0; i<nkeys; i++)
   8953 			mp->mp_ptrs[i] = copy->mp_ptrs[i];
   8954 		mp->mp_lower = copy->mp_lower;
   8955 		mp->mp_upper = copy->mp_upper;
   8956 		memcpy(NODEPTR(mp, nkeys-1), NODEPTR(copy, nkeys-1),
   8957 			env->me_psize - copy->mp_upper - PAGEBASE);
   8958 
   8959 		/* reset back to original page */
   8960 		if (newindx < split_indx) {
   8961 			mc->mc_pg[mc->mc_top] = mp;
   8962 		} else {
   8963 			mc->mc_pg[mc->mc_top] = rp;
   8964 			mc->mc_ki[ptop]++;
   8965 			/* Make sure mc_ki is still valid.
   8966 			 */
   8967 			if (mn.mc_pg[ptop] != mc->mc_pg[ptop] &&
   8968 				mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) {
   8969 				for (i=0; i<=ptop; i++) {
   8970 					mc->mc_pg[i] = mn.mc_pg[i];
   8971 					mc->mc_ki[i] = mn.mc_ki[i];
   8972 				}
   8973 			}
   8974 		}
   8975 		if (nflags & MDB_RESERVE) {
   8976 			node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
   8977 			if (!(node->mn_flags & F_BIGDATA))
   8978 				newdata->mv_data = NODEDATA(node);
   8979 		}
   8980 	} else {
   8981 		if (newindx >= split_indx) {
   8982 			mc->mc_pg[mc->mc_top] = rp;
   8983 			mc->mc_ki[ptop]++;
   8984 			/* Make sure mc_ki is still valid.
   8985 			 */
   8986 			if (mn.mc_pg[ptop] != mc->mc_pg[ptop] &&
   8987 				mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) {
   8988 				for (i=0; i<=ptop; i++) {
   8989 					mc->mc_pg[i] = mn.mc_pg[i];
   8990 					mc->mc_ki[i] = mn.mc_ki[i];
   8991 				}
   8992 			}
   8993 		}
   8994 	}
   8995 
   8996 	{
   8997 		/* Adjust other cursors pointing to mp */
   8998 		MDB_cursor *m2, *m3;
   8999 		MDB_dbi dbi = mc->mc_dbi;
   9000 		nkeys = NUMKEYS(mp);
   9001 
   9002 		for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
   9003 			if (mc->mc_flags & C_SUB)
   9004 				m3 = &m2->mc_xcursor->mx_cursor;
   9005 			else
   9006 				m3 = m2;
   9007 			if (m3 == mc)
   9008 				continue;
   9009 			if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED))
   9010 				continue;
   9011 			if (new_root) {
   9012 				int k;
   9013 				/* sub cursors may be on different DB */
   9014 				if (m3->mc_pg[0] != mp)
   9015 					continue;
   9016 				/* root split */
   9017 				for (k=new_root; k>=0; k--) {
   9018 					m3->mc_ki[k+1] = m3->mc_ki[k];
   9019 					m3->mc_pg[k+1] = m3->mc_pg[k];
   9020 				}
   9021 				if (m3->mc_ki[0] >= nkeys) {
   9022 					m3->mc_ki[0] = 1;
   9023 				} else {
   9024 					m3->mc_ki[0] = 0;
   9025 				}
   9026 				m3->mc_pg[0] = mc->mc_pg[0];
   9027 				m3->mc_snum++;
   9028 				m3->mc_top++;
   9029 			}
   9030 			if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) {
   9031 				if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE))
   9032 					m3->mc_ki[mc->mc_top]++;
   9033 				if (m3->mc_ki[mc->mc_top] >= nkeys) {
   9034 					m3->mc_pg[mc->mc_top] = rp;
   9035 					m3->mc_ki[mc->mc_top] -= nkeys;
   9036 					for (i=0; i<mc->mc_top; i++) {
   9037 						m3->mc_ki[i] = mn.mc_ki[i];
   9038 						m3->mc_pg[i] = mn.mc_pg[i];
   9039 					}
   9040 				}
   9041 			} else if (!did_split && m3->mc_top >= ptop && m3->mc_pg[ptop] == mc->mc_pg[ptop] &&
   9042 				m3->mc_ki[ptop] >= mc->mc_ki[ptop]) {
   9043 				m3->mc_ki[ptop]++;
   9044 			}
   9045 			if (IS_LEAF(mp))
   9046 				XCURSOR_REFRESH(m3, mc->mc_top, m3->mc_pg[mc->mc_top]);
   9047 		}
   9048 	}
   9049 	DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp)));
   9050 
   9051 done:
   9052 	if (copy)					/* tmp page */
   9053 		mdb_page_free(env, copy);
   9054 	if (rc)
   9055 		mc->mc_txn->mt_flags |= MDB_TXN_ERROR;
   9056 	return rc;
   9057 }
   9058 
   9059 int
   9060 mdb_put(MDB_txn *txn, MDB_dbi dbi,
   9061     MDB_val *key, MDB_val *data, unsigned int flags)
   9062 {
   9063 	MDB_cursor mc;
   9064 	MDB_xcursor mx;
   9065 	int rc;
   9066 
   9067 	if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
   9068 		return EINVAL;
   9069 
   9070 	if (flags & ~(MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP))
   9071 		return EINVAL;
   9072 
   9073 	if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
   9074 		return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
   9075 
   9076 	mdb_cursor_init(&mc, txn, dbi, &mx);
   9077 	mc.mc_next = txn->mt_cursors[dbi];
   9078 	txn->mt_cursors[dbi] = &mc;
   9079 	rc = mdb_cursor_put(&mc, key, data, flags);
   9080 	txn->mt_cursors[dbi] = mc.mc_next;
   9081 	return rc;
   9082 }
   9083 
   9084 #ifndef MDB_WBUF
   9085 #define MDB_WBUF	(1024*1024)
   9086 #endif
   9087 #define MDB_EOF		0x10	/**< #mdb_env_copyfd1() is done reading */
   9088 
   9089 	/** State needed for a double-buffering compacting copy. */
   9090 typedef struct mdb_copy {
   9091 	MDB_env *mc_env;
   9092 	MDB_txn *mc_txn;
   9093 	pthread_mutex_t mc_mutex;
   9094 	pthread_cond_t mc_cond;	/**< Condition variable for #mc_new */
   9095 	char *mc_wbuf[2];
   9096 	char *mc_over[2];
   9097 	int mc_wlen[2];
   9098 	int mc_olen[2];
   9099 	pgno_t mc_next_pgno;
   9100 	HANDLE mc_fd;
   9101 	int mc_toggle;			/**< Buffer number in provider */
   9102 	int mc_new;				/**< (0-2 buffers to write) | (#MDB_EOF at end) */
   9103 	/** Error code.  Never cleared if set.  Both threads can set nonzero
   9104 	 *	to fail the copy.  Not mutex-protected, LMDB expects atomic int.
   9105 	 */
   9106 	volatile int mc_error;
   9107 } mdb_copy;
   9108 
   9109 	/** Dedicated writer thread for compacting copy. */
   9110 static THREAD_RET ESECT CALL_CONV
   9111 mdb_env_copythr(void *arg)
   9112 {
   9113 	mdb_copy *my = arg;
   9114 	char *ptr;
   9115 	int toggle = 0, wsize, rc;
   9116 #ifdef _WIN32
   9117 	DWORD len;
   9118 #define DO_WRITE(rc, fd, ptr, w2, len)	rc = WriteFile(fd, ptr, w2, &len, NULL)
   9119 #else
   9120 	int len;
   9121 #define DO_WRITE(rc, fd, ptr, w2, len)	len = write(fd, ptr, w2); rc = (len >= 0)
   9122 #ifdef SIGPIPE
   9123 	sigset_t set;
   9124 	sigemptyset(&set);
   9125 	sigaddset(&set, SIGPIPE);
   9126 	if ((rc = pthread_sigmask(SIG_BLOCK, &set, NULL)) != 0)
   9127 		my->mc_error = rc;
   9128 #endif
   9129 #endif
   9130 
   9131 	pthread_mutex_lock(&my->mc_mutex);
   9132 	for(;;) {
   9133 		while (!my->mc_new)
   9134 			pthread_cond_wait(&my->mc_cond, &my->mc_mutex);
   9135 		if (my->mc_new == 0 + MDB_EOF) /* 0 buffers, just EOF */
   9136 			break;
   9137 		wsize = my->mc_wlen[toggle];
   9138 		ptr = my->mc_wbuf[toggle];
   9139 again:
   9140 		rc = MDB_SUCCESS;
   9141 		while (wsize > 0 && !my->mc_error) {
   9142 			DO_WRITE(rc, my->mc_fd, ptr, wsize, len);
   9143 			if (!rc) {
   9144 				rc = ErrCode();
   9145 #if defined(SIGPIPE) && !defined(_WIN32)
   9146 				if (rc == EPIPE) {
   9147 					/* Collect the pending SIGPIPE, otherwise at least OS X
   9148 					 * gives it to the process on thread-exit (ITS#8504).
   9149 					 */
   9150 					int tmp;
   9151 					sigwait(&set, &tmp);
   9152 				}
   9153 #endif
   9154 				break;
   9155 			} else if (len > 0) {
   9156 				rc = MDB_SUCCESS;
   9157 				ptr += len;
   9158 				wsize -= len;
   9159 				continue;
   9160 			} else {
   9161 				rc = EIO;
   9162 				break;
   9163 			}
   9164 		}
   9165 		if (rc) {
   9166 			my->mc_error = rc;
   9167 		}
   9168 		/* If there's an overflow page tail, write it too */
   9169 		if (my->mc_olen[toggle]) {
   9170 			wsize = my->mc_olen[toggle];
   9171 			ptr = my->mc_over[toggle];
   9172 			my->mc_olen[toggle] = 0;
   9173 			goto again;
   9174 		}
   9175 		my->mc_wlen[toggle] = 0;
   9176 		toggle ^= 1;
   9177 		/* Return the empty buffer to provider */
   9178 		my->mc_new--;
   9179 		pthread_cond_signal(&my->mc_cond);
   9180 	}
   9181 	pthread_mutex_unlock(&my->mc_mutex);
   9182 	return (THREAD_RET)0;
   9183 #undef DO_WRITE
   9184 }
   9185 
   9186 	/** Give buffer and/or #MDB_EOF to writer thread, await unused buffer.
   9187 	 *
   9188 	 * @param[in] my control structure.
   9189 	 * @param[in] adjust (1 to hand off 1 buffer) | (MDB_EOF when ending).
   9190 	 */
   9191 static int ESECT
   9192 mdb_env_cthr_toggle(mdb_copy *my, int adjust)
   9193 {
   9194 	pthread_mutex_lock(&my->mc_mutex);
   9195 	my->mc_new += adjust;
   9196 	pthread_cond_signal(&my->mc_cond);
   9197 	while (my->mc_new & 2)		/* both buffers in use */
   9198 		pthread_cond_wait(&my->mc_cond, &my->mc_mutex);
   9199 	pthread_mutex_unlock(&my->mc_mutex);
   9200 
   9201 	my->mc_toggle ^= (adjust & 1);
   9202 	/* Both threads reset mc_wlen, to be safe from threading errors */
   9203 	my->mc_wlen[my->mc_toggle] = 0;
   9204 	return my->mc_error;
   9205 }
   9206 
   9207 	/** Depth-first tree traversal for compacting copy.
   9208 	 * @param[in] my control structure.
   9209 	 * @param[in,out] pg database root.
   9210 	 * @param[in] flags includes #F_DUPDATA if it is a sorted-duplicate sub-DB.
   9211 	 */
   9212 static int ESECT
   9213 mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
   9214 {
   9215 	MDB_cursor mc = {0};
   9216 	MDB_node *ni;
   9217 	MDB_page *mo, *mp, *leaf;
   9218 	char *buf, *ptr;
   9219 	int rc, toggle;
   9220 	unsigned int i;
   9221 
   9222 	/* Empty DB, nothing to do */
   9223 	if (*pg == P_INVALID)
   9224 		return MDB_SUCCESS;
   9225 
   9226 	mc.mc_snum = 1;
   9227 	mc.mc_txn = my->mc_txn;
   9228 
   9229 	rc = mdb_page_get(&mc, *pg, &mc.mc_pg[0], NULL);
   9230 	if (rc)
   9231 		return rc;
   9232 	rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST);
   9233 	if (rc)
   9234 		return rc;
   9235 
   9236 	/* Make cursor pages writable */
   9237 	buf = ptr = malloc(my->mc_env->me_psize * mc.mc_snum);
   9238 	if (buf == NULL)
   9239 		return ENOMEM;
   9240 
   9241 	for (i=0; i<mc.mc_top; i++) {
   9242 		mdb_page_copy((MDB_page *)ptr, mc.mc_pg[i], my->mc_env->me_psize);
   9243 		mc.mc_pg[i] = (MDB_page *)ptr;
   9244 		ptr += my->mc_env->me_psize;
   9245 	}
   9246 
   9247 	/* This is writable space for a leaf page. Usually not needed. */
   9248 	leaf = (MDB_page *)ptr;
   9249 
   9250 	toggle = my->mc_toggle;
   9251 	while (mc.mc_snum > 0) {
   9252 		unsigned n;
   9253 		mp = mc.mc_pg[mc.mc_top];
   9254 		n = NUMKEYS(mp);
   9255 
   9256 		if (IS_LEAF(mp)) {
   9257 			if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) {
   9258 				for (i=0; i<n; i++) {
   9259 					ni = NODEPTR(mp, i);
   9260 					if (ni->mn_flags & F_BIGDATA) {
   9261 						MDB_page *omp;
   9262 						pgno_t pg;
   9263 
   9264 						/* Need writable leaf */
   9265 						if (mp != leaf) {
   9266 							mc.mc_pg[mc.mc_top] = leaf;
   9267 							mdb_page_copy(leaf, mp, my->mc_env->me_psize);
   9268 							mp = leaf;
   9269 							ni = NODEPTR(mp, i);
   9270 						}
   9271 
   9272 						memcpy(&pg, NODEDATA(ni), sizeof(pg));
   9273 						memcpy(NODEDATA(ni), &my->mc_next_pgno, sizeof(pgno_t));
   9274 						rc = mdb_page_get(&mc, pg, &omp, NULL);
   9275 						if (rc)
   9276 							goto done;
   9277 						if (my->mc_wlen[toggle] >= MDB_WBUF) {
   9278 							rc = mdb_env_cthr_toggle(my, 1);
   9279 							if (rc)
   9280 								goto done;
   9281 							toggle = my->mc_toggle;
   9282 						}
   9283 						mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
   9284 						memcpy(mo, omp, my->mc_env->me_psize);
   9285 						mo->mp_pgno = my->mc_next_pgno;
   9286 						my->mc_next_pgno += omp->mp_pages;
   9287 						my->mc_wlen[toggle] += my->mc_env->me_psize;
   9288 						if (omp->mp_pages > 1) {
   9289 							my->mc_olen[toggle] = my->mc_env->me_psize * (omp->mp_pages - 1);
   9290 							my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize;
   9291 							rc = mdb_env_cthr_toggle(my, 1);
   9292 							if (rc)
   9293 								goto done;
   9294 							toggle = my->mc_toggle;
   9295 						}
   9296 					} else if (ni->mn_flags & F_SUBDATA) {
   9297 						MDB_db db;
   9298 
   9299 						/* Need writable leaf */
   9300 						if (mp != leaf) {
   9301 							mc.mc_pg[mc.mc_top] = leaf;
   9302 							mdb_page_copy(leaf, mp, my->mc_env->me_psize);
   9303 							mp = leaf;
   9304 							ni = NODEPTR(mp, i);
   9305 						}
   9306 
   9307 						memcpy(&db, NODEDATA(ni), sizeof(db));
   9308 						my->mc_toggle = toggle;
   9309 						rc = mdb_env_cwalk(my, &db.md_root, ni->mn_flags & F_DUPDATA);
   9310 						if (rc)
   9311 							goto done;
   9312 						toggle = my->mc_toggle;
   9313 						memcpy(NODEDATA(ni), &db, sizeof(db));
   9314 					}
   9315 				}
   9316 			}
   9317 		} else {
   9318 			mc.mc_ki[mc.mc_top]++;
   9319 			if (mc.mc_ki[mc.mc_top] < n) {
   9320 				pgno_t pg;
   9321 again:
   9322 				ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]);
   9323 				pg = NODEPGNO(ni);
   9324 				rc = mdb_page_get(&mc, pg, &mp, NULL);
   9325 				if (rc)
   9326 					goto done;
   9327 				mc.mc_top++;
   9328 				mc.mc_snum++;
   9329 				mc.mc_ki[mc.mc_top] = 0;
   9330 				if (IS_BRANCH(mp)) {
   9331 					/* Whenever we advance to a sibling branch page,
   9332 					 * we must proceed all the way down to its first leaf.
   9333 					 */
   9334 					mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize);
   9335 					goto again;
   9336 				} else
   9337 					mc.mc_pg[mc.mc_top] = mp;
   9338 				continue;
   9339 			}
   9340 		}
   9341 		if (my->mc_wlen[toggle] >= MDB_WBUF) {
   9342 			rc = mdb_env_cthr_toggle(my, 1);
   9343 			if (rc)
   9344 				goto done;
   9345 			toggle = my->mc_toggle;
   9346 		}
   9347 		mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
   9348 		mdb_page_copy(mo, mp, my->mc_env->me_psize);
   9349 		mo->mp_pgno = my->mc_next_pgno++;
   9350 		my->mc_wlen[toggle] += my->mc_env->me_psize;
   9351 		if (mc.mc_top) {
   9352 			/* Update parent if there is one */
   9353 			ni = NODEPTR(mc.mc_pg[mc.mc_top-1], mc.mc_ki[mc.mc_top-1]);
   9354 			SETPGNO(ni, mo->mp_pgno);
   9355 			mdb_cursor_pop(&mc);
   9356 		} else {
   9357 			/* Otherwise we're done */
   9358 			*pg = mo->mp_pgno;
   9359 			break;
   9360 		}
   9361 	}
   9362 done:
   9363 	free(buf);
   9364 	return rc;
   9365 }
   9366 
   9367 	/** Copy environment with compaction. */
   9368 static int ESECT
   9369 mdb_env_copyfd1(MDB_env *env, HANDLE fd)
   9370 {
   9371 	MDB_meta *mm;
   9372 	MDB_page *mp;
   9373 	mdb_copy my = {0};
   9374 	MDB_txn *txn = NULL;
   9375 	pthread_t thr;
   9376 	pgno_t root, new_root;
   9377 	int rc = MDB_SUCCESS;
   9378 
   9379 #ifdef _WIN32
   9380 	if (!(my.mc_mutex = CreateMutex(NULL, FALSE, NULL)) ||
   9381 		!(my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL))) {
   9382 		rc = ErrCode();
   9383 		goto done;
   9384 	}
   9385 	my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize);
   9386 	if (my.mc_wbuf[0] == NULL) {
   9387 		/* _aligned_malloc() sets errno, but we use Windows error codes */
   9388 		rc = ERROR_NOT_ENOUGH_MEMORY;
   9389 		goto done;
   9390 	}
   9391 #else
   9392 	if ((rc = pthread_mutex_init(&my.mc_mutex, NULL)) != 0)
   9393 		return rc;
   9394 	if ((rc = pthread_cond_init(&my.mc_cond, NULL)) != 0)
   9395 		goto done2;
   9396 #ifdef HAVE_MEMALIGN
   9397 	my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2);
   9398 	if (my.mc_wbuf[0] == NULL) {
   9399 		rc = errno;
   9400 		goto done;
   9401 	}
   9402 #else
   9403 	{
   9404 		void *p;
   9405 		if ((rc = posix_memalign(&p, env->me_os_psize, MDB_WBUF*2)) != 0)
   9406 			goto done;
   9407 		my.mc_wbuf[0] = p;
   9408 	}
   9409 #endif
   9410 #endif
   9411 	memset(my.mc_wbuf[0], 0, MDB_WBUF*2);
   9412 	my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF;
   9413 	my.mc_next_pgno = NUM_METAS;
   9414 	my.mc_env = env;
   9415 	my.mc_fd = fd;
   9416 	rc = THREAD_CREATE(thr, mdb_env_copythr, &my);
   9417 	if (rc)
   9418 		goto done;
   9419 
   9420 	rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
   9421 	if (rc)
   9422 		goto finish;
   9423 
   9424 	mp = (MDB_page *)my.mc_wbuf[0];
   9425 	memset(mp, 0, NUM_METAS * env->me_psize);
   9426 	mp->mp_pgno = 0;
   9427 	mp->mp_flags = P_META;
   9428 	mm = (MDB_meta *)METADATA(mp);
   9429 	mdb_env_init_meta0(env, mm);
   9430 	mm->mm_address = env->me_metas[0]->mm_address;
   9431 
   9432 	mp = (MDB_page *)(my.mc_wbuf[0] + env->me_psize);
   9433 	mp->mp_pgno = 1;
   9434 	mp->mp_flags = P_META;
   9435 	*(MDB_meta *)METADATA(mp) = *mm;
   9436 	mm = (MDB_meta *)METADATA(mp);
   9437 
   9438 	/* Set metapage 1 with current main DB */
   9439 	root = new_root = txn->mt_dbs[MAIN_DBI].md_root;
   9440 	if (root != P_INVALID) {
   9441 		/* Count free pages + freeDB pages.  Subtract from last_pg
   9442 		 * to find the new last_pg, which also becomes the new root.
   9443 		 */
   9444 		MDB_ID freecount = 0;
   9445 		MDB_cursor mc;
   9446 		MDB_val key, data;
   9447 		mdb_cursor_init(&mc, txn, FREE_DBI, NULL);
   9448 		while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0)
   9449 			freecount += *(MDB_ID *)data.mv_data;
   9450 		if (rc != MDB_NOTFOUND)
   9451 			goto finish;
   9452 		freecount += txn->mt_dbs[FREE_DBI].md_branch_pages +
   9453 			txn->mt_dbs[FREE_DBI].md_leaf_pages +
   9454 			txn->mt_dbs[FREE_DBI].md_overflow_pages;
   9455 
   9456 		new_root = txn->mt_next_pgno - 1 - freecount;
   9457 		mm->mm_last_pg = new_root;
   9458 		mm->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI];
   9459 		mm->mm_dbs[MAIN_DBI].md_root = new_root;
   9460 	} else {
   9461 		/* When the DB is empty, handle it specially to
   9462 		 * fix any breakage like page leaks from ITS#8174.
   9463 		 */
   9464 		mm->mm_dbs[MAIN_DBI].md_flags = txn->mt_dbs[MAIN_DBI].md_flags;
   9465 	}
   9466 	if (root != P_INVALID || mm->mm_dbs[MAIN_DBI].md_flags) {
   9467 		mm->mm_txnid = 1;		/* use metapage 1 */
   9468 	}
   9469 
   9470 	my.mc_wlen[0] = env->me_psize * NUM_METAS;
   9471 	my.mc_txn = txn;
   9472 	rc = mdb_env_cwalk(&my, &root, 0);
   9473 	if (rc == MDB_SUCCESS && root != new_root) {
   9474 		rc = MDB_INCOMPATIBLE;	/* page leak or corrupt DB */
   9475 	}
   9476 
   9477 finish:
   9478 	if (rc)
   9479 		my.mc_error = rc;
   9480 	mdb_env_cthr_toggle(&my, 1 | MDB_EOF);
   9481 	rc = THREAD_FINISH(thr);
   9482 	mdb_txn_abort(txn);
   9483 
   9484 done:
   9485 #ifdef _WIN32
   9486 	if (my.mc_wbuf[0]) _aligned_free(my.mc_wbuf[0]);
   9487 	if (my.mc_cond)  CloseHandle(my.mc_cond);
   9488 	if (my.mc_mutex) CloseHandle(my.mc_mutex);
   9489 #else
   9490 	free(my.mc_wbuf[0]);
   9491 	pthread_cond_destroy(&my.mc_cond);
   9492 done2:
   9493 	pthread_mutex_destroy(&my.mc_mutex);
   9494 #endif
   9495 	return rc ? rc : my.mc_error;
   9496 }
   9497 
   9498 	/** Copy environment as-is. */
   9499 static int ESECT
   9500 mdb_env_copyfd0(MDB_env *env, HANDLE fd)
   9501 {
   9502 	MDB_txn *txn = NULL;
   9503 	mdb_mutexref_t wmutex = NULL;
   9504 	int rc;
   9505 	size_t wsize, w3;
   9506 	char *ptr;
   9507 #ifdef _WIN32
   9508 	DWORD len, w2;
   9509 #define DO_WRITE(rc, fd, ptr, w2, len)	rc = WriteFile(fd, ptr, w2, &len, NULL)
   9510 #else
   9511 	ssize_t len;
   9512 	size_t w2;
   9513 #define DO_WRITE(rc, fd, ptr, w2, len)	len = write(fd, ptr, w2); rc = (len >= 0)
   9514 #endif
   9515 
   9516 	/* Do the lock/unlock of the reader mutex before starting the
   9517 	 * write txn.  Otherwise other read txns could block writers.
   9518 	 */
   9519 	rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
   9520 	if (rc)
   9521 		return rc;
   9522 
   9523 	if (env->me_txns) {
   9524 		/* We must start the actual read txn after blocking writers */
   9525 		mdb_txn_end(txn, MDB_END_RESET_TMP);
   9526 
   9527 		/* Temporarily block writers until we snapshot the meta pages */
   9528 		wmutex = env->me_wmutex;
   9529 		if (LOCK_MUTEX(rc, env, wmutex))
   9530 			goto leave;
   9531 
   9532 		rc = mdb_txn_renew0(txn);
   9533 		if (rc) {
   9534 			UNLOCK_MUTEX(wmutex);
   9535 			goto leave;
   9536 		}
   9537 	}
   9538 
   9539 	wsize = env->me_psize * NUM_METAS;
   9540 	ptr = env->me_map;
   9541 	w2 = wsize;
   9542 	while (w2 > 0) {
   9543 		DO_WRITE(rc, fd, ptr, w2, len);
   9544 		if (!rc) {
   9545 			rc = ErrCode();
   9546 			break;
   9547 		} else if (len > 0) {
   9548 			rc = MDB_SUCCESS;
   9549 			ptr += len;
   9550 			w2 -= len;
   9551 			continue;
   9552 		} else {
   9553 			/* Non-blocking or async handles are not supported */
   9554 			rc = EIO;
   9555 			break;
   9556 		}
   9557 	}
   9558 	if (wmutex)
   9559 		UNLOCK_MUTEX(wmutex);
   9560 
   9561 	if (rc)
   9562 		goto leave;
   9563 
   9564 	w3 = txn->mt_next_pgno * env->me_psize;
   9565 	{
   9566 		size_t fsize = 0;
   9567 		if ((rc = mdb_fsize(env->me_fd, &fsize)))
   9568 			goto leave;
   9569 		if (w3 > fsize)
   9570 			w3 = fsize;
   9571 	}
   9572 	wsize = w3 - wsize;
   9573 	while (wsize > 0) {
   9574 		if (wsize > MAX_WRITE)
   9575 			w2 = MAX_WRITE;
   9576 		else
   9577 			w2 = wsize;
   9578 		DO_WRITE(rc, fd, ptr, w2, len);
   9579 		if (!rc) {
   9580 			rc = ErrCode();
   9581 			break;
   9582 		} else if (len > 0) {
   9583 			rc = MDB_SUCCESS;
   9584 			ptr += len;
   9585 			wsize -= len;
   9586 			continue;
   9587 		} else {
   9588 			rc = EIO;
   9589 			break;
   9590 		}
   9591 	}
   9592 
   9593 leave:
   9594 	mdb_txn_abort(txn);
   9595 	return rc;
   9596 }
   9597 
   9598 int ESECT
   9599 mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags)
   9600 {
   9601 	if (flags & MDB_CP_COMPACT)
   9602 		return mdb_env_copyfd1(env, fd);
   9603 	else
   9604 		return mdb_env_copyfd0(env, fd);
   9605 }
   9606 
   9607 int ESECT
   9608 mdb_env_copyfd(MDB_env *env, HANDLE fd)
   9609 {
   9610 	return mdb_env_copyfd2(env, fd, 0);
   9611 }
   9612 
   9613 int ESECT
   9614 mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
   9615 {
   9616 	int rc;
   9617 	MDB_name fname;
   9618 	HANDLE newfd = INVALID_HANDLE_VALUE;
   9619 
   9620 	rc = mdb_fname_init(path, env->me_flags | MDB_NOLOCK, &fname);
   9621 	if (rc == MDB_SUCCESS) {
   9622 		rc = mdb_fopen(env, &fname, MDB_O_COPY, 0666, &newfd);
   9623 		mdb_fname_destroy(fname);
   9624 	}
   9625 	if (rc == MDB_SUCCESS) {
   9626 		rc = mdb_env_copyfd2(env, newfd, flags);
   9627 		if (close(newfd) < 0 && rc == MDB_SUCCESS)
   9628 			rc = ErrCode();
   9629 	}
   9630 	return rc;
   9631 }
   9632 
   9633 int ESECT
   9634 mdb_env_copy(MDB_env *env, const char *path)
   9635 {
   9636 	return mdb_env_copy2(env, path, 0);
   9637 }
   9638 
   9639 int ESECT
   9640 mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff)
   9641 {
   9642 	if (flag & ~CHANGEABLE)
   9643 		return EINVAL;
   9644 	if (onoff)
   9645 		env->me_flags |= flag;
   9646 	else
   9647 		env->me_flags &= ~flag;
   9648 	return MDB_SUCCESS;
   9649 }
   9650 
   9651 int ESECT
   9652 mdb_env_get_flags(MDB_env *env, unsigned int *arg)
   9653 {
   9654 	if (!env || !arg)
   9655 		return EINVAL;
   9656 
   9657 	*arg = env->me_flags & (CHANGEABLE|CHANGELESS);
   9658 	return MDB_SUCCESS;
   9659 }
   9660 
   9661 int ESECT
   9662 mdb_env_set_userctx(MDB_env *env, void *ctx)
   9663 {
   9664 	if (!env)
   9665 		return EINVAL;
   9666 	env->me_userctx = ctx;
   9667 	return MDB_SUCCESS;
   9668 }
   9669 
   9670 void * ESECT
   9671 mdb_env_get_userctx(MDB_env *env)
   9672 {
   9673 	return env ? env->me_userctx : NULL;
   9674 }
   9675 
   9676 int ESECT
   9677 mdb_env_set_assert(MDB_env *env, MDB_assert_func *func)
   9678 {
   9679 	if (!env)
   9680 		return EINVAL;
   9681 #ifndef NDEBUG
   9682 	env->me_assert_func = func;
   9683 #endif
   9684 	return MDB_SUCCESS;
   9685 }
   9686 
   9687 int ESECT
   9688 mdb_env_get_path(MDB_env *env, const char **arg)
   9689 {
   9690 	if (!env || !arg)
   9691 		return EINVAL;
   9692 
   9693 	*arg = env->me_path;
   9694 	return MDB_SUCCESS;
   9695 }
   9696 
   9697 int ESECT
   9698 mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *arg)
   9699 {
   9700 	if (!env || !arg)
   9701 		return EINVAL;
   9702 
   9703 	*arg = env->me_fd;
   9704 	return MDB_SUCCESS;
   9705 }
   9706 
   9707 /** Common code for #mdb_stat() and #mdb_env_stat().
   9708  * @param[in] env the environment to operate in.
   9709  * @param[in] db the #MDB_db record containing the stats to return.
   9710  * @param[out] arg the address of an #MDB_stat structure to receive the stats.
   9711  * @return 0, this function always succeeds.
   9712  */
   9713 static int ESECT
   9714 mdb_stat0(MDB_env *env, MDB_db *db, MDB_stat *arg)
   9715 {
   9716 	arg->ms_psize = env->me_psize;
   9717 	arg->ms_depth = db->md_depth;
   9718 	arg->ms_branch_pages = db->md_branch_pages;
   9719 	arg->ms_leaf_pages = db->md_leaf_pages;
   9720 	arg->ms_overflow_pages = db->md_overflow_pages;
   9721 	arg->ms_entries = db->md_entries;
   9722 
   9723 	return MDB_SUCCESS;
   9724 }
   9725 
   9726 int ESECT
   9727 mdb_env_stat(MDB_env *env, MDB_stat *arg)
   9728 {
   9729 	MDB_meta *meta;
   9730 
   9731 	if (env == NULL || arg == NULL)
   9732 		return EINVAL;
   9733 
   9734 	meta = mdb_env_pick_meta(env);
   9735 
   9736 	return mdb_stat0(env, &meta->mm_dbs[MAIN_DBI], arg);
   9737 }
   9738 
   9739 int ESECT
   9740 mdb_env_info(MDB_env *env, MDB_envinfo *arg)
   9741 {
   9742 	MDB_meta *meta;
   9743 
   9744 	if (env == NULL || arg == NULL)
   9745 		return EINVAL;
   9746 
   9747 	meta = mdb_env_pick_meta(env);
   9748 	arg->me_mapaddr = meta->mm_address;
   9749 	arg->me_last_pgno = meta->mm_last_pg;
   9750 	arg->me_last_txnid = meta->mm_txnid;
   9751 
   9752 	arg->me_mapsize = env->me_mapsize;
   9753 	arg->me_maxreaders = env->me_maxreaders;
   9754 	arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : 0;
   9755 	return MDB_SUCCESS;
   9756 }
   9757 
   9758 /** Set the default comparison functions for a database.
   9759  * Called immediately after a database is opened to set the defaults.
   9760  * The user can then override them with #mdb_set_compare() or
   9761  * #mdb_set_dupsort().
   9762  * @param[in] txn A transaction handle returned by #mdb_txn_begin()
   9763  * @param[in] dbi A database handle returned by #mdb_dbi_open()
   9764  */
   9765 static void
   9766 mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi)
   9767 {
   9768 	uint16_t f = txn->mt_dbs[dbi].md_flags;
   9769 
   9770 	txn->mt_dbxs[dbi].md_cmp =
   9771 		(f & MDB_REVERSEKEY) ? mdb_cmp_memnr :
   9772 		(f & MDB_INTEGERKEY) ? mdb_cmp_cint  : mdb_cmp_memn;
   9773 
   9774 	txn->mt_dbxs[dbi].md_dcmp =
   9775 		!(f & MDB_DUPSORT) ? 0 :
   9776 		((f & MDB_INTEGERDUP)
   9777 		 ? ((f & MDB_DUPFIXED)   ? mdb_cmp_int   : mdb_cmp_cint)
   9778 		 : ((f & MDB_REVERSEDUP) ? mdb_cmp_memnr : mdb_cmp_memn));
   9779 }
   9780 
   9781 int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi)
   9782 {
   9783 	MDB_val key, data;
   9784 	MDB_dbi i;
   9785 	MDB_cursor mc;
   9786 	MDB_db dummy;
   9787 	int rc, dbflag, exact;
   9788 	unsigned int unused = 0, seq;
   9789 	char *namedup;
   9790 	size_t len;
   9791 
   9792 	if (flags & ~VALID_FLAGS)
   9793 		return EINVAL;
   9794 	if (txn->mt_flags & MDB_TXN_BLOCKED)
   9795 		return MDB_BAD_TXN;
   9796 
   9797 	/* main DB? */
   9798 	if (!name) {
   9799 		*dbi = MAIN_DBI;
   9800 		if (flags & PERSISTENT_FLAGS) {
   9801 			uint16_t f2 = flags & PERSISTENT_FLAGS;
   9802 			/* make sure flag changes get committed */
   9803 			if ((txn->mt_dbs[MAIN_DBI].md_flags | f2) != txn->mt_dbs[MAIN_DBI].md_flags) {
   9804 				txn->mt_dbs[MAIN_DBI].md_flags |= f2;
   9805 				txn->mt_flags |= MDB_TXN_DIRTY;
   9806 			}
   9807 		}
   9808 		mdb_default_cmp(txn, MAIN_DBI);
   9809 		return MDB_SUCCESS;
   9810 	}
   9811 
   9812 	if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) {
   9813 		mdb_default_cmp(txn, MAIN_DBI);
   9814 	}
   9815 
   9816 	/* Is the DB already open? */
   9817 	len = strlen(name);
   9818 	for (i=CORE_DBS; i<txn->mt_numdbs; i++) {
   9819 		if (!txn->mt_dbxs[i].md_name.mv_size) {
   9820 			/* Remember this free slot */
   9821 			if (!unused) unused = i;
   9822 			continue;
   9823 		}
   9824 		if (len == txn->mt_dbxs[i].md_name.mv_size &&
   9825 			!strncmp(name, txn->mt_dbxs[i].md_name.mv_data, len)) {
   9826 			*dbi = i;
   9827 			return MDB_SUCCESS;
   9828 		}
   9829 	}
   9830 
   9831 	/* If no free slot and max hit, fail */
   9832 	if (!unused && txn->mt_numdbs >= txn->mt_env->me_maxdbs)
   9833 		return MDB_DBS_FULL;
   9834 
   9835 	/* Cannot mix named databases with some mainDB flags */
   9836 	if (txn->mt_dbs[MAIN_DBI].md_flags & (MDB_DUPSORT|MDB_INTEGERKEY))
   9837 		return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND;
   9838 
   9839 	/* Find the DB info */
   9840 	dbflag = DB_NEW|DB_VALID|DB_USRVALID;
   9841 	exact = 0;
   9842 	key.mv_size = len;
   9843 	key.mv_data = (void *)name;
   9844 	mdb_cursor_init(&mc, txn, MAIN_DBI, NULL);
   9845 	rc = mdb_cursor_set(&mc, &key, &data, MDB_SET, &exact);
   9846 	if (rc == MDB_SUCCESS) {
   9847 		/* make sure this is actually a DB */
   9848 		MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]);
   9849 		if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA)
   9850 			return MDB_INCOMPATIBLE;
   9851 	} else {
   9852 		if (rc != MDB_NOTFOUND || !(flags & MDB_CREATE))
   9853 			return rc;
   9854 		if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY))
   9855 			return EACCES;
   9856 	}
   9857 
   9858 	/* Done here so we cannot fail after creating a new DB */
   9859 	if ((namedup = strdup(name)) == NULL)
   9860 		return ENOMEM;
   9861 
   9862 	if (rc) {
   9863 		/* MDB_NOTFOUND and MDB_CREATE: Create new DB */
   9864 		data.mv_size = sizeof(MDB_db);
   9865 		data.mv_data = &dummy;
   9866 		memset(&dummy, 0, sizeof(dummy));
   9867 		dummy.md_root = P_INVALID;
   9868 		dummy.md_flags = flags & PERSISTENT_FLAGS;
   9869 		WITH_CURSOR_TRACKING(mc,
   9870 			rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA));
   9871 		dbflag |= DB_DIRTY;
   9872 	}
   9873 
   9874 	if (rc) {
   9875 		free(namedup);
   9876 	} else {
   9877 		/* Got info, register DBI in this txn */
   9878 		unsigned int slot = unused ? unused : txn->mt_numdbs;
   9879 		txn->mt_dbxs[slot].md_name.mv_data = namedup;
   9880 		txn->mt_dbxs[slot].md_name.mv_size = len;
   9881 		txn->mt_dbxs[slot].md_rel = NULL;
   9882 		txn->mt_dbflags[slot] = dbflag;
   9883 		/* txn-> and env-> are the same in read txns, use
   9884 		 * tmp variable to avoid undefined assignment
   9885 		 */
   9886 		seq = ++txn->mt_env->me_dbiseqs[slot];
   9887 		txn->mt_dbiseqs[slot] = seq;
   9888 
   9889 		memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db));
   9890 		*dbi = slot;
   9891 		mdb_default_cmp(txn, slot);
   9892 		if (!unused) {
   9893 			txn->mt_numdbs++;
   9894 		}
   9895 	}
   9896 
   9897 	return rc;
   9898 }
   9899 
   9900 int ESECT
   9901 mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg)
   9902 {
   9903 	if (!arg || !TXN_DBI_EXIST(txn, dbi, DB_VALID))
   9904 		return EINVAL;
   9905 
   9906 	if (txn->mt_flags & MDB_TXN_BLOCKED)
   9907 		return MDB_BAD_TXN;
   9908 
   9909 	if (txn->mt_dbflags[dbi] & DB_STALE) {
   9910 		MDB_cursor mc;
   9911 		MDB_xcursor mx;
   9912 		/* Stale, must read the DB's root. cursor_init does it for us. */
   9913 		mdb_cursor_init(&mc, txn, dbi, &mx);
   9914 	}
   9915 	return mdb_stat0(txn->mt_env, &txn->mt_dbs[dbi], arg);
   9916 }
   9917 
   9918 void mdb_dbi_close(MDB_env *env, MDB_dbi dbi)
   9919 {
   9920 	char *ptr;
   9921 	if (dbi < CORE_DBS || dbi >= env->me_maxdbs)
   9922 		return;
   9923 	ptr = env->me_dbxs[dbi].md_name.mv_data;
   9924 	/* If there was no name, this was already closed */
   9925 	if (ptr) {
   9926 		env->me_dbxs[dbi].md_name.mv_data = NULL;
   9927 		env->me_dbxs[dbi].md_name.mv_size = 0;
   9928 		env->me_dbflags[dbi] = 0;
   9929 		env->me_dbiseqs[dbi]++;
   9930 		free(ptr);
   9931 	}
   9932 }
   9933 
   9934 int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags)
   9935 {
   9936 	/* We could return the flags for the FREE_DBI too but what's the point? */
   9937 	if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
   9938 		return EINVAL;
   9939 	*flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS;
   9940 	return MDB_SUCCESS;
   9941 }
   9942 
   9943 /** Add all the DB's pages to the free list.
   9944  * @param[in] mc Cursor on the DB to free.
   9945  * @param[in] subs non-Zero to check for sub-DBs in this DB.
   9946  * @return 0 on success, non-zero on failure.
   9947  */
   9948 static int
   9949 mdb_drop0(MDB_cursor *mc, int subs)
   9950 {
   9951 	int rc;
   9952 
   9953 	rc = mdb_page_search(mc, NULL, MDB_PS_FIRST);
   9954 	if (rc == MDB_SUCCESS) {
   9955 		MDB_txn *txn = mc->mc_txn;
   9956 		MDB_node *ni;
   9957 		MDB_cursor mx;
   9958 		unsigned int i;
   9959 
   9960 		/* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves.
   9961 		 * This also avoids any P_LEAF2 pages, which have no nodes.
   9962 		 * Also if the DB doesn't have sub-DBs and has no overflow
   9963 		 * pages, omit scanning leaves.
   9964 		 */
   9965 		if ((mc->mc_flags & C_SUB) ||
   9966 			(!subs && !mc->mc_db->md_overflow_pages))
   9967 			mdb_cursor_pop(mc);
   9968 
   9969 		mdb_cursor_copy(mc, &mx);
   9970 		while (mc->mc_snum > 0) {
   9971 			MDB_page *mp = mc->mc_pg[mc->mc_top];
   9972 			unsigned n = NUMKEYS(mp);
   9973 			if (IS_LEAF(mp)) {
   9974 				for (i=0; i<n; i++) {
   9975 					ni = NODEPTR(mp, i);
   9976 					if (ni->mn_flags & F_BIGDATA) {
   9977 						MDB_page *omp;
   9978 						pgno_t pg;
   9979 						memcpy(&pg, NODEDATA(ni), sizeof(pg));
   9980 						rc = mdb_page_get(mc, pg, &omp, NULL);
   9981 						if (rc != 0)
   9982 							goto done;
   9983 						mdb_cassert(mc, IS_OVERFLOW(omp));
   9984 						rc = mdb_midl_append_range(&txn->mt_free_pgs,
   9985 							pg, omp->mp_pages);
   9986 						if (rc)
   9987 							goto done;
   9988 						mc->mc_db->md_overflow_pages -= omp->mp_pages;
   9989 						if (!mc->mc_db->md_overflow_pages && !subs)
   9990 							break;
   9991 					} else if (subs && (ni->mn_flags & F_SUBDATA)) {
   9992 						mdb_xcursor_init1(mc, ni);
   9993 						rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0);
   9994 						if (rc)
   9995 							goto done;
   9996 					}
   9997 				}
   9998 				if (!subs && !mc->mc_db->md_overflow_pages)
   9999 					goto pop;
  10000 			} else {
  10001 				if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0)
  10002 					goto done;
  10003 				for (i=0; i<n; i++) {
  10004 					pgno_t pg;
  10005 					ni = NODEPTR(mp, i);
  10006 					pg = NODEPGNO(ni);
  10007 					/* free it */
  10008 					mdb_midl_xappend(txn->mt_free_pgs, pg);
  10009 				}
  10010 			}
  10011 			if (!mc->mc_top)
  10012 				break;
  10013 			mc->mc_ki[mc->mc_top] = i;
  10014 			rc = mdb_cursor_sibling(mc, 1);
  10015 			if (rc) {
  10016 				if (rc != MDB_NOTFOUND)
  10017 					goto done;
  10018 				/* no more siblings, go back to beginning
  10019 				 * of previous level.
  10020 				 */
  10021 pop:
  10022 				mdb_cursor_pop(mc);
  10023 				mc->mc_ki[0] = 0;
  10024 				for (i=1; i<mc->mc_snum; i++) {
  10025 					mc->mc_ki[i] = 0;
  10026 					mc->mc_pg[i] = mx.mc_pg[i];
  10027 				}
  10028 			}
  10029 		}
  10030 		/* free it */
  10031 		rc = mdb_midl_append(&txn->mt_free_pgs, mc->mc_db->md_root);
  10032 done:
  10033 		if (rc)
  10034 			txn->mt_flags |= MDB_TXN_ERROR;
  10035 	} else if (rc == MDB_NOTFOUND) {
  10036 		rc = MDB_SUCCESS;
  10037 	}
  10038 	mc->mc_flags &= ~C_INITIALIZED;
  10039 	return rc;
  10040 }
  10041 
  10042 int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
  10043 {
  10044 	MDB_cursor *mc, *m2;
  10045 	int rc;
  10046 
  10047 	if ((unsigned)del > 1 || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
  10048 		return EINVAL;
  10049 
  10050 	if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY))
  10051 		return EACCES;
  10052 
  10053 	if (TXN_DBI_CHANGED(txn, dbi))
  10054 		return MDB_BAD_DBI;
  10055 
  10056 	rc = mdb_cursor_open(txn, dbi, &mc);
  10057 	if (rc)
  10058 		return rc;
  10059 
  10060 	rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT);
  10061 	/* Invalidate the dropped DB's cursors */
  10062 	for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next)
  10063 		m2->mc_flags &= ~(C_INITIALIZED|C_EOF);
  10064 	if (rc)
  10065 		goto leave;
  10066 
  10067 	/* Can't delete the main DB */
  10068 	if (del && dbi >= CORE_DBS) {
  10069 		rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, F_SUBDATA);
  10070 		if (!rc) {
  10071 			txn->mt_dbflags[dbi] = DB_STALE;
  10072 			mdb_dbi_close(txn->mt_env, dbi);
  10073 		} else {
  10074 			txn->mt_flags |= MDB_TXN_ERROR;
  10075 		}
  10076 	} else {
  10077 		/* reset the DB record, mark it dirty */
  10078 		txn->mt_dbflags[dbi] |= DB_DIRTY;
  10079 		txn->mt_dbs[dbi].md_depth = 0;
  10080 		txn->mt_dbs[dbi].md_branch_pages = 0;
  10081 		txn->mt_dbs[dbi].md_leaf_pages = 0;
  10082 		txn->mt_dbs[dbi].md_overflow_pages = 0;
  10083 		txn->mt_dbs[dbi].md_entries = 0;
  10084 		txn->mt_dbs[dbi].md_root = P_INVALID;
  10085 
  10086 		txn->mt_flags |= MDB_TXN_DIRTY;
  10087 	}
  10088 leave:
  10089 	mdb_cursor_close(mc);
  10090 	return rc;
  10091 }
  10092 
  10093 int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
  10094 {
  10095 	if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
  10096 		return EINVAL;
  10097 
  10098 	txn->mt_dbxs[dbi].md_cmp = cmp;
  10099 	return MDB_SUCCESS;
  10100 }
  10101 
  10102 int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
  10103 {
  10104 	if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
  10105 		return EINVAL;
  10106 
  10107 	txn->mt_dbxs[dbi].md_dcmp = cmp;
  10108 	return MDB_SUCCESS;
  10109 }
  10110 
  10111 int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel)
  10112 {
  10113 	if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
  10114 		return EINVAL;
  10115 
  10116 	txn->mt_dbxs[dbi].md_rel = rel;
  10117 	return MDB_SUCCESS;
  10118 }
  10119 
  10120 int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx)
  10121 {
  10122 	if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
  10123 		return EINVAL;
  10124 
  10125 	txn->mt_dbxs[dbi].md_relctx = ctx;
  10126 	return MDB_SUCCESS;
  10127 }
  10128 
  10129 int ESECT
  10130 mdb_env_get_maxkeysize(MDB_env *env)
  10131 {
  10132 	return ENV_MAXKEY(env);
  10133 }
  10134 
  10135 int ESECT
  10136 mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx)
  10137 {
  10138 	unsigned int i, rdrs;
  10139 	MDB_reader *mr;
  10140 	char buf[64];
  10141 	int rc = 0, first = 1;
  10142 
  10143 	if (!env || !func)
  10144 		return -1;
  10145 	if (!env->me_txns) {
  10146 		return func("(no reader locks)\n", ctx);
  10147 	}
  10148 	rdrs = env->me_txns->mti_numreaders;
  10149 	mr = env->me_txns->mti_readers;
  10150 	for (i=0; i<rdrs; i++) {
  10151 		if (mr[i].mr_pid) {
  10152 			txnid_t	txnid = mr[i].mr_txnid;
  10153 			sprintf(buf, txnid == (txnid_t)-1 ?
  10154 				"%10d %"Z"x -\n" : "%10d %"Z"x %"Z"u\n",
  10155 				(int)mr[i].mr_pid, (size_t)mr[i].mr_tid, txnid);
  10156 			if (first) {
  10157 				first = 0;
  10158 				rc = func("    pid     thread     txnid\n", ctx);
  10159 				if (rc < 0)
  10160 					break;
  10161 			}
  10162 			rc = func(buf, ctx);
  10163 			if (rc < 0)
  10164 				break;
  10165 		}
  10166 	}
  10167 	if (first) {
  10168 		rc = func("(no active readers)\n", ctx);
  10169 	}
  10170 	return rc;
  10171 }
  10172 
  10173 /** Insert pid into list if not already present.
  10174  * return -1 if already present.
  10175  */
  10176 static int ESECT
  10177 mdb_pid_insert(MDB_PID_T *ids, MDB_PID_T pid)
  10178 {
  10179 	/* binary search of pid in list */
  10180 	unsigned base = 0;
  10181 	unsigned cursor = 1;
  10182 	int val = 0;
  10183 	unsigned n = ids[0];
  10184 
  10185 	while( 0 < n ) {
  10186 		unsigned pivot = n >> 1;
  10187 		cursor = base + pivot + 1;
  10188 		val = pid - ids[cursor];
  10189 
  10190 		if( val < 0 ) {
  10191 			n = pivot;
  10192 
  10193 		} else if ( val > 0 ) {
  10194 			base = cursor;
  10195 			n -= pivot + 1;
  10196 
  10197 		} else {
  10198 			/* found, so it's a duplicate */
  10199 			return -1;
  10200 		}
  10201 	}
  10202 
  10203 	if( val > 0 ) {
  10204 		++cursor;
  10205 	}
  10206 	ids[0]++;
  10207 	for (n = ids[0]; n > cursor; n--)
  10208 		ids[n] = ids[n-1];
  10209 	ids[n] = pid;
  10210 	return 0;
  10211 }
  10212 
  10213 int ESECT
  10214 mdb_reader_check(MDB_env *env, int *dead)
  10215 {
  10216 	if (!env)
  10217 		return EINVAL;
  10218 	if (dead)
  10219 		*dead = 0;
  10220 	return env->me_txns ? mdb_reader_check0(env, 0, dead) : MDB_SUCCESS;
  10221 }
  10222 
  10223 /** As #mdb_reader_check(). \b rlocked is set if caller locked #me_rmutex. */
  10224 static int ESECT
  10225 mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
  10226 {
  10227 	mdb_mutexref_t rmutex = rlocked ? NULL : env->me_rmutex;
  10228 	unsigned int i, j, rdrs;
  10229 	MDB_reader *mr;
  10230 	MDB_PID_T *pids, pid;
  10231 	int rc = MDB_SUCCESS, count = 0;
  10232 
  10233 	rdrs = env->me_txns->mti_numreaders;
  10234 	pids = malloc((rdrs+1) * sizeof(MDB_PID_T));
  10235 	if (!pids)
  10236 		return ENOMEM;
  10237 	pids[0] = 0;
  10238 	mr = env->me_txns->mti_readers;
  10239 	for (i=0; i<rdrs; i++) {
  10240 		pid = mr[i].mr_pid;
  10241 		if (pid && pid != env->me_pid) {
  10242 			if (mdb_pid_insert(pids, pid) == 0) {
  10243 				if (!mdb_reader_pid(env, Pidcheck, pid)) {
  10244 					/* Stale reader found */
  10245 					j = i;
  10246 					if (rmutex) {
  10247 						if ((rc = LOCK_MUTEX0(rmutex)) != 0) {
  10248 							if ((rc = mdb_mutex_failed(env, rmutex, rc)))
  10249 								break;
  10250 							rdrs = 0; /* the above checked all readers */
  10251 						} else {
  10252 							/* Recheck, a new process may have reused pid */
  10253 							if (mdb_reader_pid(env, Pidcheck, pid))
  10254 								j = rdrs;
  10255 						}
  10256 					}
  10257 					for (; j<rdrs; j++)
  10258 							if (mr[j].mr_pid == pid) {
  10259 								DPRINTF(("clear stale reader pid %u txn %"Z"d",
  10260 									(unsigned) pid, mr[j].mr_txnid));
  10261 								mr[j].mr_pid = 0;
  10262 								count++;
  10263 							}
  10264 					if (rmutex)
  10265 						UNLOCK_MUTEX(rmutex);
  10266 				}
  10267 			}
  10268 		}
  10269 	}
  10270 	free(pids);
  10271 	if (dead)
  10272 		*dead = count;
  10273 	return rc;
  10274 }
  10275 
  10276 #ifdef MDB_ROBUST_SUPPORTED
  10277 /** Handle #LOCK_MUTEX0() failure.
  10278  * Try to repair the lock file if the mutex owner died.
  10279  * @param[in] env	the environment handle
  10280  * @param[in] mutex	LOCK_MUTEX0() mutex
  10281  * @param[in] rc	LOCK_MUTEX0() error (nonzero)
  10282  * @return 0 on success with the mutex locked, or an error code on failure.
  10283  */
  10284 static int ESECT
  10285 mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc)
  10286 {
  10287 	int rlocked, rc2;
  10288 	MDB_meta *meta;
  10289 
  10290 	if (rc == MDB_OWNERDEAD) {
  10291 		/* We own the mutex. Clean up after dead previous owner. */
  10292 		rc = MDB_SUCCESS;
  10293 		rlocked = (mutex == env->me_rmutex);
  10294 		if (!rlocked) {
  10295 			/* Keep mti_txnid updated, otherwise next writer can
  10296 			 * overwrite data which latest meta page refers to.
  10297 			 */
  10298 			meta = mdb_env_pick_meta(env);
  10299 			env->me_txns->mti_txnid = meta->mm_txnid;
  10300 			/* env is hosed if the dead thread was ours */
  10301 			if (env->me_txn) {
  10302 				env->me_flags |= MDB_FATAL_ERROR;
  10303 				env->me_txn = NULL;
  10304 				rc = MDB_PANIC;
  10305 			}
  10306 		}
  10307 		DPRINTF(("%cmutex owner died, %s", (rlocked ? 'r' : 'w'),
  10308 			(rc ? "this process' env is hosed" : "recovering")));
  10309 		rc2 = mdb_reader_check0(env, rlocked, NULL);
  10310 		if (rc2 == 0)
  10311 			rc2 = mdb_mutex_consistent(mutex);
  10312 		if (rc || (rc = rc2)) {
  10313 			DPRINTF(("LOCK_MUTEX recovery failed, %s", mdb_strerror(rc)));
  10314 			UNLOCK_MUTEX(mutex);
  10315 		}
  10316 	} else {
  10317 #ifdef _WIN32
  10318 		rc = ErrCode();
  10319 #endif
  10320 		DPRINTF(("LOCK_MUTEX failed, %s", mdb_strerror(rc)));
  10321 	}
  10322 
  10323 	return rc;
  10324 }
  10325 #endif	/* MDB_ROBUST_SUPPORTED */
  10326 
  10327 #if defined(_WIN32)
  10328 /** Convert \b src to new wchar_t[] string with room for \b xtra extra chars */
  10329 static int ESECT
  10330 utf8_to_utf16(const char *src, MDB_name *dst, int xtra)
  10331 {
  10332 	int rc, need = 0;
  10333 	wchar_t *result = NULL;
  10334 	for (;;) {					/* malloc result, then fill it in */
  10335 		need = MultiByteToWideChar(CP_UTF8, 0, src, -1, result, need);
  10336 		if (!need) {
  10337 			rc = ErrCode();
  10338 			free(result);
  10339 			return rc;
  10340 		}
  10341 		if (!result) {
  10342 			result = malloc(sizeof(wchar_t) * (need + xtra));
  10343 			if (!result)
  10344 				return ENOMEM;
  10345 			continue;
  10346 		}
  10347 		dst->mn_alloced = 1;
  10348 		dst->mn_len = need - 1;
  10349 		dst->mn_val = result;
  10350 		return MDB_SUCCESS;
  10351 	}
  10352 }
  10353 #endif /* defined(_WIN32) */
  10354 /** @} */