myvtm/Framework/libtoolkit/shm_mem.c

#include "precompile.h"
#include "toolkit.h"
#include "q_malloc.h"
#include "shm_mem.h"
#include "shm.h"
#include "strutil.h"
#include "memutil.h"
#include "modCheck.h"
#include <stdarg.h>
#include <fcntl.h>

#ifndef _WIN32
#include <sys/mman.h>
#include <unistd.h>
#endif //NOT _WIN32

#define TAG TOOLKIT_TAG("shm_mem")

#	define MY_MALLOC qm_malloc
#	define MY_FREE qm_free

#define SHM_MEM_MSIZE (SHM_MEM_SIZE*1024*1024)

//#define USE_ANON_MMAP


#ifndef _WIN32
#define USE_PARENT_FIXED_ADDR
#define SHM_MEM_ADDR_NAME "rvc.shm.spshell.addr"

#define USE_DEVSHM_MMAP
#define SHM_MEM_NAME "rvc.shm.spshell"
#define SHM_MMAP
#endif //NOT _WIN32

#	define  shm_malloc_init qm_malloc_init

//实体间共享
TOOLKIT_API struct qm_block* shm_block = NULL;

static int shm_shmid=-1; /*shared memory id*/

static void* shm_mempool= INVALID_MAP;

TOOLKIT_API void shm_lock()
{
	qm_lock(shm_block);
}

TOOLKIT_API void shm_unlock()
{
	qm_unlock(shm_block);
}

/** 
https://stackoverflow.com/questions/5939578/how-to-choose-a-fixed-address-for-shared-memory-mapping
*/


/* look at a buffer if there is perhaps enough space for the new size
   (It is beneficial to do so because vq_malloc is pretty stateful
    and if we ask for a new buffer size, we can still make it happy
    with current buffer); if so, we return current buffer again;
    otherwise, we free it, allocate a new one and return it; no
    guarantee for buffer content; if allocation fails, we return
    NULL
*/

TOOLKIT_API int shm_getmem(int key, void** hint, int newcreate)
{
    int fd = -1;
    int flag = 0;

#ifndef _WIN32
    const size_t region_size = sysconf(_SC_PAGE_SIZE);
    WLog_DBG(TAG, "allocating SHM block, page size: %d", region_size);
#endif //NOT _WIN32

#ifdef SHM_MMAP
	if (shm_mempool && (shm_mempool != INVALID_MAP)) {
#else
    if ((shm_shmid != -1) || (shm_mempool != INVALID_MAP)) {
#endif
        WLog_WARN(TAG, "%s: already initialized.", __FUNCTION__);
        return 0;
    }

#ifdef SHM_MMAP

#ifndef USE_ANON_MMAP

#ifdef USE_DEVSHM_MMAP
    WLog_WARN(TAG, "using /dev/shm memory");
    flag = O_RDWR;
    if (newcreate) {
        if (0 != shm_unlink(SHM_MEM_NAME)) {
            WLog_WARN(TAG, "could not unlink  %s: %d, %s", SHM_MEM_NAME, errno, strerror(errno));
            flag |= O_CREAT;
        } else {
            flag |= (O_CREAT | O_EXCL);
        }
    }

    do {
        mode_t fm = umask(0);
        fd = shm_open(SHM_MEM_NAME, flag, 0777);
        umask(fm);
    } while (0);

    if (fd == -1) {
        WLog_ERR(TAG, "could not open %s: %d, %s", SHM_MEM_NAME, errno, strerror(errno));
        return -1;
    }
#else
    WLog_WARN(TAG, "using /dev/zero memory");
    fd = open("/dev/zero", O_RDWR);
    if (fd == -1) {
        WLog_ERR(TAG, "could not open /dev/zero: %d %s", errno, strerror(errno));
        return -1;
    }
#endif // USE_DEVSHM_MMAP

#else
    WLog_WARN(TAG, "using anonyous memory");
#endif /* USE_ANON_MMAP */

#ifdef USE_DEVSHM_MMAP
    //for share memory object, this method would set specified length capacity.
    if (newcreate)  ftruncate(fd, SHM_MEM_MSIZE);
#endif // USE_DEVSHM_MMAP

	shm_mempool = osips_shm_getmem(fd, hint ? *hint : NULL, SHM_MEM_MSIZE);

#ifndef USE_ANON_MMAP
    close(fd);
#endif /* USE_ANON_MMAP */

    if (shm_mempool == INVALID_MAP) {
		WLog_ERR(TAG, "could not attach shared memory segment: %d, %s", errno, strerror(errno));
        /* destroy segment*/
        if (newcreate) shm_mem_destroy();
        return -1;
    }
#else
    if (newcreate) {
        int ts = 0;
        do 
        {
            shm_shmid = shmget(key, SHM_MEM_MSIZE, IPC_CREAT | IPC_EXCL);
            if (shm_shmid == -1 && errno == EEXIST) {
                shm_shmid = shmget(key, SHM_MEM_MSIZE, 0);
                osips_shm_mem_destroy();
            } else {  break; }
        } while ((++ts) < 2);
    } else {
        shm_shmid = shmget(key, SHM_MEM_MSIZE, 0);
    }
    if (shm_shmid == -1) {
        WLog_ERR(TAG, "shmget failed, error: %d, %s", errno, strerror(errno));
        return -1;
    }
    /*link to its address after creating share memory */
    shm_mempool = shmat(shm_shmid, hint ? *hint : NULL, 0);
    if (shm_mempool == INVALID_MAP) {
        /* destroy segment*/
        WLog_ERR(TAG, "shmat failed, error: %d %s", errno, strerror(errno));
        /**
        Invalid shmid value
            unaligned (i.e., not page-aligned and SHM_RND was not specified)
            or invalid shmaddr value, or can't attach segment at shmaddr,
            or SHM_REMAP was specified and shmaddr was NULL.
        */
        if(newcreate) shm_mem_destroy();
        return -1;
    }
#endif
    if (hint) {
        WLog_DBG(TAG, "attach addr: %p", shm_mempool);
        *hint = shm_mempool;
    }
    return 0;
}

static int shm_mem_init_mallocs(void* mempool, unsigned long pool_size, int newcreate)
{
#if !defined(NO_FIXED_ADDR)
    if (newcreate) {
        /* init it for qmalloc*/
        shm_block = qm_malloc_init((char*)mempool, pool_size, newcreate);
        if (shm_block == 0) {
            WLog_ERR(TAG, "shm malloc init failed!");
            shm_mem_destroy();
            return -1;
        }
    } else {
        WLog_DBG(TAG, "enrich shm block with mempool %p", mempool);
        shm_block = (struct qm_block*)mempool;
        WLog_DBG(TAG, "to print block size:");
        WLog_DBG(TAG, "\t\t%d", shm_block->size);
    }
#else
    shm_block = qm_malloc_init((char*)mempool, pool_size, newcreate);
    if (shm_block == 0) {
        WLog_ERR(TAG, "shm malloc init failed!");
        shm_mem_destroy2(newcreate);
        return -1;
    }
#endif
	return 0;
}

TOOLKIT_API void* shm_mem_init(void* hint, int newcreate)
{
#if 0
    if (!newcreate && hint != NULL) {
        shm_mempool = hint;
        WLog_WARN(TAG, "set shm memory directly: %p", shm_mempool);
        if (shm_mem_init_mallocs(shm_mempool, SHM_MEM_MSIZE, newcreate) < 0) {
            shm_mempool = NULL;
            return NULL;
        }
    }
#endif //USE_PARENT_FIXED_ADDR

    if (shm_getmem(-1, &hint, newcreate) < 0
        || shm_mem_init_mallocs(shm_mempool, SHM_MEM_MSIZE, newcreate) < 0) {
        return NULL;
    }
    return hint;
}

TOOLKIT_API int shm_mem_init2(int key, void *hint, int newcreate)
{
	int ret;
    ret = shm_getmem(key, &hint, newcreate);
    if (ret < 0) {
        WLog_ERR(TAG, "get mem failed: %d", ret);
        return ret;
    }
    /*actually: hint equals with shm_mempool*/
    ret = shm_mem_init_mallocs(hint/*shm_mempool*/, SHM_MEM_MSIZE, newcreate);

#ifdef USE_PARENT_FIXED_ADDR
    if (ret == 0 && !!newcreate) {
        mode_t fm = umask(0);
        int fd = shm_open(SHM_MEM_ADDR_NAME, O_CREAT | O_TRUNC | O_RDWR, 0777);
        umask(fm);
        if (fd != -1) {
            ftruncate(fd, sizeof(unsigned long));
            void** addr_ptr = (void**)mmap(0, sizeof(unsigned long), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
            close(fd);
            if (addr_ptr != INVALID_MAP) {
                *addr_ptr = hint;
                WLog_DBG(TAG, "write mmap: %p %p", addr_ptr, *addr_ptr);
                msync(addr_ptr, sizeof(unsigned long), MS_SYNC);
                munmap(addr_ptr, sizeof(unsigned long));
            } else {
                WLog_ERR(TAG, "failed to mmap: %d, %s", errno, strerror(errno));
                shm_mem_destroy();
                ret = -2;
            }
        } else {
            WLog_ERR(TAG, "shm open failed: %s, %d, %s", SHM_MEM_ADDR_NAME, errno, strerror(errno));
            shm_mem_destroy();
            ret = -2;
        }
    }
#endif
    return ret;
}

int shm_getmem_addr(void** hint)
{
#ifdef USE_PARENT_FIXED_ADDR
    mode_t fm = umask(0);
    int fd = shm_open(SHM_MEM_ADDR_NAME, O_RDONLY, 0777);
    umask(fm);
    if (fd != -1) {
        lseek(fd, 0, SEEK_SET);
        void** addr_ptr = (void**)mmap(NULL, sizeof(unsigned long), PROT_READ, MAP_SHARED, fd, 0);
        close(fd);
        if (addr_ptr != INVALID_MAP) {
            if (hint) { *hint = *addr_ptr; }
            WLog_DBG(TAG, "read mmap: %p, %p", addr_ptr, *addr_ptr);
            munmap(addr_ptr, sizeof(unsigned long));
            return 0;
        } else {
            WLog_ERR(TAG, "failed to mmap: %d, %s", errno, strerror(errno));
            return TOOLKIT_EAI_BADHINTS;
        }
    } else {
        WLog_ERR(TAG, "shm open failed: %s, %d, %s", SHM_MEM_ADDR_NAME, errno, strerror(errno));
        return TOOLKIT_EIO;
    }
    return TOOLKIT_UNKNOWN;
#else
    return TOOLKIT_ESRCH;
#endif // USE_PARENT_FIXED_ADDR
}

TOOLKIT_API void shm_mem_destroy2(int newcreate)
{
    if (newcreate) {
        osips_shm_mem_destroy();
    } else {
        osips_shm_relmem(shm_mempool, SHM_MEM_MSIZE);
        shm_mempool = INVALID_MAP;
    }
}

TOOLKIT_API void shm_mem_destroy(void)
{
#if 0
    struct shmid_ds shm_info;
    if (shm_mempool && (shm_mempool != (void*)-1)) {
        shmdt(shm_mempool);
        shm_mempool = (void*)-1;
    }

    if (shm_shmid != -1) {
        /*IPC_RMID: delete share memory from system.*/
        shmctl(shm_shmid, IPC_RMID, &shm_info);
        shm_shmid = -1;
    }
#else
    osips_shm_mem_destroy();
#endif
}

TOOLKIT_API char *shm_strdup(const char *str)
{
	int n;
	char *ret;
	if (!str)
		return NULL;
	n = strlen(str);
	ret = shm_malloc(n + 1);
	if (ret) {
		memcpy(ret, str, n+1);
	}
	return ret;
}

TOOLKIT_API char *shm_strdup_printf(const char *format, ...)
{
	va_list arg;
	char *ret;
	va_start(arg, format);
	ret = shm_strdup_vprintf(format, arg);
	va_end(arg);
	return ret;
}

TOOLKIT_API char *shm_strdup_vprintf(const char *format, va_list args)
{
	char *t = strdup_vprintf(format, args);
	char *r = shm_strdup(t); 
	free(t);
	return r;
}

TOOLKIT_API void shm_set_user_data(int idx, void *user_data)
{
	qm_set_user_data(shm_block, idx, user_data);
}

TOOLKIT_API void *shm_get_user_data(int idx)
{
    void* result = NULL;
	if (shm_block == NULL) {
		WLog_ERR(TAG, "shm block is nullptr !!");
		return result;
	} 
	result = qm_get_user_data(shm_block, idx);
	if (result == NULL) {
        WLog_ERR(TAG, "to get user data null at idx %d from qm %p", idx, shm_block);
	}
	return result;
}

TOOLKIT_API void* shm_malloc(unsigned int size)
{
	void *p;
	shm_lock();
	p=shm_malloc_unsafe(size);
    if(p && size > 0) memset(p, 0, size);
	shm_unlock();

	return p; 
}



TOOLKIT_API void *shm_malloc_unsafe(unsigned int _size)
{
	return qm_malloc(shm_block, (_size));
}

TOOLKIT_API void shm_free_unsafe(void * _p )
{
		qm_free(shm_block, (_p));
}

/*
 * Allocates memory using mmap or sysv shmap
 *  - fd: a handler to a file descriptor pointing to a map file
 *  - force_addr: force mapping to a specific address
 *  - size: how large the mmap should be
 */
TOOLKIT_API void* osips_shm_getmem(int fd, void* force_addr, unsigned long size)
{
    void* ret_addr = NULL;
    int flags;
	off_t offset, pa_offset;
	size_t length;

	WLog_DBG(TAG, "Enter %s function", __FUNCTION__);

#ifdef SHM_MMAP
    flags = MAP_SHARED; //child process share memory
	if (force_addr) {
		flags |= MAP_FIXED;
		WLog_WARN(TAG, "with force address: %p", force_addr);
	}
	if (fd == -1) {
		flags |= MAP_ANONYMOUS;
	}
    WLog_DBG(TAG, "to mmap...");
    ret_addr = mmap(force_addr, size, PROT_READ | PROT_WRITE, flags, fd, 0);
    WLog_DBG(TAG, "memory address: %p", ret_addr);
#else /* USE_MMAP */
    /* TODO: handle persistent storage for SysV */
	WLog_WARN(TAG, "Cannot have persistent storage using SysV");
        if (force_addr || fd == -1)
            return INVALID_MAP;
	/** if key existed, error occur or create new one*/
    shm_shmid = shmget(IPC_PRIVATE, SHM_MEM_MSIZE, IPC_CREAT | IPC_EXCL | IPC_NOWAIT);
    if (shm_shmid == -1) {
        WLog_ERR(TAG, "could not allocate shared memory segment: %s", strerror(errno));
        return INVALID_MAP;
    }
    shm_mempool = shmat(shm_shmid, 0, 0);
#endif
    return ret_addr;
}

TOOLKIT_API void osips_shm_mem_destroy()
{
#ifndef SHM_MMAP
    struct shmid_ds shm_info;
#endif

    osips_shm_relmem(shm_mempool, SHM_MEM_MSIZE);
    shm_mempool = INVALID_MAP;

#ifndef SHM_MMAP
    if (shm_shmid != -1) {
        shmctl(shm_shmid, IPC_RMID, &shm_info);
        shm_shmid = -1;
    }
#endif

#ifdef USE_DEVSHM_MMAP
    shm_unlink(SHM_MEM_NAME);
#endif // USE_DEVSHM_MMAP

}

TOOLKIT_API void osips_shm_relmem(void* mempool, unsigned long size)
{
    if (mempool && (mempool != INVALID_MAP)) {
#ifdef SHM_MMAP
        munmap(mempool, size);
#else
        shmdt(mempool);
#endif
    }
}