Skip to content

Commit

Permalink
release logio storage for extents on file unlink
Browse files Browse the repository at this point in the history 
Each server is responsible for releasing the local extents for the
target file. For extents that have not yet been synced to the local
server, the client must release the storage.

Also includes a bug fix for extents that happened to span the
shmem and spill portions of the log. The bug was revealed by the
new library API test that makes sure we can properly reclaim
storage for deleted files (t/api/storage-reuse.c).
  • Loading branch information
@MichaelBrim @adammoody
MichaelBrim authored and adammoody committed Jul 30, 2021
1 parent 90da150 commit fc7cb5b
Show file tree
Hide file tree
Showing 9 changed files with 419 additions and 55 deletions.
18 changes: 18 additions & 0 deletions client/src/unifyfs_fid.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,24 @@ static int fid_storage_free(unifyfs_client* client,
unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(client, fid);
if ((meta != NULL) && (meta->fid == fid)) {
if (meta->storage == FILE_STORAGE_LOGIO) {
/* client needs to release unsynced write extents, since server
* does not know about them */
seg_tree_rdlock(&meta->extents_sync);
struct seg_tree_node* node = NULL;
while ((node = seg_tree_iter(&meta->extents_sync, node))) {
size_t nbytes = (size_t) (node->end - node->start + 1);
off_t log_offset = (off_t) node->ptr;
int rc = unifyfs_logio_free(client->state.logio_ctx,
log_offset, nbytes);
if (UNIFYFS_SUCCESS != rc) {
LOGERR("failed to free logio allocation for "
"client[%d:%d] log_offset=%zu nbytes=%zu",
client->state.app_id, client->state.client_id,
log_offset, nbytes);
}
}
seg_tree_unlock(&meta->extents_sync);

/* Free our write seg_tree */
seg_tree_destroy(&meta->extents_sync);

Expand Down
55 changes: 48 additions & 7 deletions common/src/slotmap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,8 @@ int slotmap_clear(slot_map* smap)

/* set used to zero */
smap->used_slots = 0;
smap->first_used_slot = -1;
smap->last_used_slot = -1;

/* zero-out use map */
uint8_t* usemap = get_use_map(smap);
Expand Down Expand Up @@ -228,9 +230,9 @@ ssize_t slotmap_reserve(slot_map* smap,

/* search for contiguous free slots */
size_t search_start = 0;
if (slot_bytes > 1) {
/* skip past (likely) used slots */
search_start = SLOT_BYTE(smap->used_slots);
if ((smap->last_used_slot != -1) && (slot_bytes > 1)) {
/* skip past likely-used slots */
search_start = SLOT_BYTE(smap->last_used_slot);
}
uint8_t* usemap = get_use_map(smap);
size_t map_bytes = slot_map_bytes(smap->total_slots);
Expand Down Expand Up @@ -292,8 +294,17 @@ ssize_t slotmap_reserve(slot_map* smap,

if (found_start) {
/* success, reserve bits in consecutive slots */
for (size_t i = 0; i < num_slots; i++) {
use_slot(usemap, start_slot + i);
size_t end_slot = start_slot + num_slots - 1;
for (size_t i = start_slot; i <= end_slot; i++) {
use_slot(usemap, i);
}
if ((smap->first_used_slot == -1) ||
(start_slot < smap->first_used_slot)) {
smap->first_used_slot = start_slot;
}
if ((smap->last_used_slot == -1) ||
(end_slot > smap->last_used_slot)) {
smap->last_used_slot = end_slot;
}
smap->used_slots += num_slots;
return (ssize_t)start_slot;
Expand Down Expand Up @@ -328,11 +339,41 @@ int slotmap_release(slot_map* smap,
}

/* release the slots */
for (size_t i = 0; i < num_slots; i++) {
release_slot(usemap, start_index + i);
size_t end_slot = start_index + num_slots - 1;
for (size_t i = start_index; i <= end_slot; i++) {
release_slot(usemap, i);
}
smap->used_slots -= num_slots;

if (smap->used_slots == 0) {
smap->first_used_slot = -1;
smap->last_used_slot = -1;
return UNIFYFS_SUCCESS;
}

/* find new first-used slot if necessary */
if (start_index == smap->first_used_slot) {
ssize_t first_slot = end_slot + 1;
while ((first_slot < smap->total_slots) &&
(!check_slot(usemap, (size_t)first_slot))) {
first_slot++;
}
if (first_slot == smap->total_slots) {
first_slot = -1;
}
smap->last_used_slot = first_slot;
}

/* find new last-used slot if necessary */
if (end_slot == smap->last_used_slot) {
ssize_t last_slot = start_index - 1;
while ((last_slot >= 0) &&
(!check_slot(usemap, (size_t)last_slot))) {
last_slot--;
}
smap->last_used_slot = last_slot;
}

return UNIFYFS_SUCCESS;
}

Expand Down
2 changes: 2 additions & 0 deletions common/src/slotmap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ extern "C" {
typedef struct slot_map {
size_t total_slots;
size_t used_slots;
ssize_t first_used_slot;
ssize_t last_used_slot;
} slot_map;

/* The slot usage bitmap immediately follows the structure in memory.
Expand Down
51 changes: 35 additions & 16 deletions common/src/unifyfs_logio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ typedef struct log_header {
size_t data_sz; /* total data bytes in log */
size_t reserved_sz; /* reserved data bytes */
size_t chunk_sz; /* data chunk size */
size_t max_reserved_slot; /* slot index for last reserved chunk */
off_t data_offset; /* file/memory offset where data chunks start */
} log_header;
/* chunk slot_map immediately follows header and occupies rest of the page */
Expand Down Expand Up @@ -136,7 +135,10 @@ static int get_spillfile(const char* path,
}

/* map log header (1st page) of spill file given by file descriptor */
static void* map_spillfile(int spill_fd, int mmap_prot, int n_pages)
static void* map_spillfile(int spill_fd,
int mmap_prot,
int n_pages,
int server)
{
int err;
size_t pgsz = get_page_size();
Expand All @@ -152,7 +154,7 @@ static void* map_spillfile(int spill_fd, int mmap_prot, int n_pages)
return NULL;
}

if (mmap_prot == PROT_READ) { /* server maps for read only */
if (server) {
log_header* loghdr = (log_header*) addr;
size_t hdr_sz = loghdr->hdr_sz;
if (hdr_sz > mapsz) {
Expand Down Expand Up @@ -184,6 +186,7 @@ int unifyfs_logio_init_server(const int app_id,
}
*pctx = NULL;

log_header* hdr = NULL;
shm_context* shm_ctx = NULL;
if (mem_size) {
/* attach to client shmem region */
Expand All @@ -195,6 +198,9 @@ int unifyfs_logio_init_server(const int app_id,
LOGERR("Failed to attach logio shmem buffer!");
return UNIFYFS_ERROR_SHMEM;
}
hdr = (log_header*) shm_ctx->addr;
LOGDBG("shmem header - hdr_sz=%zu, data_sz=%zu, data_offset=%zu",
hdr->hdr_sz, hdr->data_sz, hdr->data_offset);
}

char spillfile[UNIFYFS_MAX_FILENAME];
Expand All @@ -215,12 +221,16 @@ int unifyfs_logio_init_server(const int app_id,
return UNIFYFS_FAILURE;
} else {
/* map the start of the spill-over file, which contains log header
* and chunk slot_map. server only needs read access. */
spill_mapping = map_spillfile(spill_fd, PROT_READ, 1);
* and chunk slot_map. server needs read and write access */
int map_flags = PROT_READ | PROT_WRITE;
spill_mapping = map_spillfile(spill_fd, map_flags, 1, 1);
if (NULL == spill_mapping) {
LOGERR("Failed to map logio spill file header!");
return UNIFYFS_FAILURE;
}
hdr = (log_header*) spill_mapping;
LOGDBG("spill header - hdr_sz=%zu, data_sz=%zu, data_offset=%zu",
hdr->hdr_sz, hdr->data_sz, hdr->data_offset);
}
}

Expand Down Expand Up @@ -278,8 +288,8 @@ static int init_log_header(char* log_region,
}

/* chunk data starts after header pages */
data_size = region_size - hdr_size;
size_t n_chunks = data_size / chunk_size;
size_t data_space = region_size - hdr_size;
size_t n_chunks = data_space / chunk_size;

/* try to init chunk slotmap */
size_t slotmap_size = hdr_size - sizeof(log_header);
Expand All @@ -290,6 +300,10 @@ static int init_log_header(char* log_region,
hdr_pages++;
continue;
}

/* the data_size is an exact multiple of chunk_size, which may be
* slightly less than the data_space */
data_size = n_chunks * chunk_size;
break;
}

Expand Down Expand Up @@ -355,6 +369,9 @@ int unifyfs_logio_init_client(const int app_id,
LOGERR("Failed to initialize shmem logio header");
return rc;
}
log_header* hdr = (log_header*) memlog;
LOGDBG("shmem header - hdr_sz=%zu, data_sz=%zu, data_offset=%zu",
hdr->hdr_sz, hdr->data_sz, hdr->data_offset);
}

/* will we use spillover to store the files? */
Expand Down Expand Up @@ -405,7 +422,7 @@ int unifyfs_logio_init_client(const int app_id,
/* map start of the spill-over file, which contains log header
* and chunk slot_map. client needs read and write access. */
int map_flags = PROT_READ | PROT_WRITE;
spill_mapping = map_spillfile(spill_fd, map_flags, n_pages);
spill_mapping = map_spillfile(spill_fd, map_flags, n_pages, 0);
if (NULL == spill_mapping) {
LOGERR("Failed to map logio spill file header!");
return UNIFYFS_FAILURE;
Expand All @@ -418,6 +435,9 @@ int unifyfs_logio_init_client(const int app_id,
LOGERR("Failed to initialize spill logio header");
return rc;
}
log_header* hdr = (log_header*) spill;
LOGDBG("spill header - hdr_sz=%zu, data_sz=%zu, data_offset=%zu",
hdr->hdr_sz, hdr->data_sz, hdr->data_offset);
}
}

Expand Down Expand Up @@ -541,7 +561,6 @@ int unifyfs_logio_alloc(logio_context* ctx,
/* success, all needed chunks allocated in shmem */
allocated_bytes = res_chunks * chunk_sz;
shmem_hdr->reserved_sz += allocated_bytes;
shmem_hdr->max_reserved_slot = (res_slot + res_chunks) - 1;
res_off = (off_t)(res_slot * chunk_sz);
*log_offset = res_off;
return UNIFYFS_SUCCESS;
Expand All @@ -550,7 +569,7 @@ int unifyfs_logio_alloc(logio_context* ctx,
/* could not get full allocation in shmem, reserve any available
* chunks at the end of the shmem log */
size_t log_end_chunks = chunkmap->total_slots -
(shmem_hdr->max_reserved_slot + 1);
(chunkmap->last_used_slot + 1);
if (log_end_chunks > 0) {
res_chunks = log_end_chunks;
res_slot = slotmap_reserve(chunkmap, res_chunks);
Expand Down Expand Up @@ -584,7 +603,6 @@ int unifyfs_logio_alloc(logio_context* ctx,
if (0 == mem_res_at_end) {
/* success, full reservation in spill */
spill_hdr->reserved_sz += allocated_bytes;
spill_hdr->max_reserved_slot = (res_slot + res_chunks) - 1;
res_off = (off_t)(res_slot * chunk_sz);
if (NULL != shmem_hdr) {
/* update log offset to account for shmem log size */
Expand Down Expand Up @@ -622,9 +640,8 @@ int unifyfs_logio_alloc(logio_context* ctx,
res_slot = slotmap_reserve(chunkmap, res_chunks);
if (-1 != res_slot) {
/* success, full reservation in spill */
allocated_bytes = res_chunks * chunk_sz;
spill_hdr->reserved_sz += allocated_bytes;
spill_hdr->max_reserved_slot =
(res_slot + res_chunks) - 1;
res_off = (off_t)(res_slot * chunk_sz);
if (NULL != shmem_hdr) {
/* update log offset to include shmem log size */
Expand All @@ -636,10 +653,7 @@ int unifyfs_logio_alloc(logio_context* ctx,
} else {
/* successful reservation spanning shmem and spill */
shmem_hdr->reserved_sz += mem_allocation;
shmem_hdr->max_reserved_slot =
(mem_res_slot + mem_res_nchk) - 1;
spill_hdr->reserved_sz += allocated_bytes;
spill_hdr->max_reserved_slot = (res_slot + res_chunks) - 1;
*log_offset = res_off;
return UNIFYFS_SUCCESS;
}
Expand All @@ -656,6 +670,7 @@ int unifyfs_logio_alloc(logio_context* ctx,
LOGERR("slotmap_release() for logio shmem failed");
}
}
LOGDBG("returning ENOSPC");
return ENOSPC;
}

Expand Down Expand Up @@ -689,6 +704,8 @@ int unifyfs_logio_free(logio_context* ctx,
off_t spill_offset = 0;
get_log_sizes(log_offset, nbytes, mem_size,
&sz_in_mem, &sz_in_spill, &spill_offset);
LOGDBG("log_off=%zu, nbytes=%zu : mem_sz=%zu spill_sz=%zu spill_off=%zu",
log_offset, nbytes, sz_in_mem, sz_in_spill, (size_t)spill_offset);

int rc = UNIFYFS_SUCCESS;
size_t chunk_sz, chunk_slot, num_chunks;
Expand Down Expand Up @@ -828,6 +845,8 @@ int unifyfs_logio_write(logio_context* ctx,
off_t spill_offset = 0;
get_log_sizes(log_offset, nbytes, mem_size,
&sz_in_mem, &sz_in_spill, &spill_offset);
LOGDBG("log_off=%zu, nbytes=%zu : mem_sz=%zu spill_sz=%zu spill_off=%zu",
log_offset, nbytes, sz_in_mem, sz_in_spill, (size_t)spill_offset);

/* do writes */
int err_rc = 0;
Expand Down
Loading

0 comments on commit fc7cb5b

Please sign in to comment.