system.h File Reference

#include <stdint.h>
#include <mpi.h>
#include <stdarg.h>

Include dependency graph for system.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros
#define	MPI_UINT64   MPI_UNSIGNED_LONG
#define	MPI_INT64   MPI_LONG
#define	MPIU_Barrier(comm)   _MPIU_Barrier(__FILE__, __LINE__, comm)

Functions
int	cluster_get_num_hosts (void)
double	get_physmem_bytes (void)
double	get_random_number (uint64_t id)
void	set_random_numbers (int seed)
void	sumup_large_ints (int n, int *src, int64_t *res)
void	sumup_longs (int n, int64_t *src, int64_t *res)
int64_t	count_sum (int64_t countLocal)
int	MPIU_Any (int condition, MPI_Comm comm)
void	MPIU_write_pids (char *filename)
size_t	gadget_compact_thread_arrays (int *dest, int *srcs[], size_t sizes[], int narrays)
void	gadget_setup_thread_arrays (int *dest, int *srcs[], size_t sizes[], size_t total_size, int narrays)
int	MPI_Alltoallv_smart (void *sendbuf, int *sendcnts, int *sdispls, MPI_Datatype sendtype, void *recvbuf, int *recvcnts, int *rdispls, MPI_Datatype recvtype, MPI_Comm comm)
int	MPI_Alltoallv_sparse (void *sendbuf, int *sendcnts, int *sdispls, MPI_Datatype sendtype, void *recvbuf, int *recvcnts, int *rdispls, MPI_Datatype recvtype, MPI_Comm comm)
double	timediff (double t0, double t1)
double	second (void)
size_t	sizemax (size_t a, size_t b)
static int64_t	atomic_fetch_and_add_64 (int64_t *ptr, int64_t value)
static int	atomic_fetch_and_add (int *ptr, int value)
static int	atomic_add_and_fetch (int *ptr, int value)
void	MPIU_Trace (MPI_Comm comm, int where, const char *fmt,...)
void	MPIU_Tracev (MPI_Comm comm, int where, int error, const char *fmt, va_list va)
int	_MPIU_Barrier (const char *fn, const int ln, MPI_Comm comm)

Macro Definition Documentation

MPI_INT64

#define MPI_INT64   MPI_LONG

Definition at line 12 of file system.h.

MPI_UINT64

#define MPI_UINT64   MPI_UNSIGNED_LONG

Definition at line 11 of file system.h.

MPIU_Barrier

#define MPIU_Barrier

(

comm

)

_MPIU_Barrier(__FILE__, __LINE__, comm)

Definition at line 103 of file system.h.

Function Documentation

_MPIU_Barrier()

int _MPIU_Barrier	(	const char *	fn,
		const int	line,
		MPI_Comm	comm
	)

A fancy MPI barrier (use MPIU_Barrier macro)

• aborts if barrier mismatch occurs
• warn if some ranks are very imbalanced.

Definition at line 500 of file system.c.

{
    int ThisTask, NTask;
    MPI_Comm_size(comm, &NTask);
    MPI_Comm_rank(comm, &ThisTask);
    int * recvbuf = ta_malloc("tags", int, NTask);
    int tag = 0;
    int i;
    for(i = 0; fn[i]; i ++) {
        tag += (int)fn[i] * 8;
    }
    tag += line;
 
    MPI_Request request;
    MPI_Igather(&tag, 1, MPI_INT, recvbuf, 1, MPI_INT, 0, comm, &request);
    i = 0;
    int flag = 1;
    int tsleep = 0;
    while(flag) {
        MPI_Test(&request, &flag, MPI_STATUS_IGNORE);
        if(flag) break;
        usleep(i * 1000);
        tsleep += i * 1000;
        i = i + 1;
        if(i == 50) {
            if(ThisTask == 0) {
                MPIU_Trace(comm, 0, "Waited more than %g seconds during barrier %s : %d \n", tsleep / 1000000., fn, line);
            }
            break;
        }
    }
    MPI_Wait(&request, MPI_STATUS_IGNORE);
    /* now check if all ranks indeed hit the same barrier. Some MPIs do allow them to mix up! */
    if (ThisTask == 0) {
        for(i = 0; i < NTask; i ++) {
            if(recvbuf[i] != tag) {
                MPIU_Trace(comm, 0, "Task %d Did not hit barrier at %s : %d; expecting %d, got %d\n", i, fn, line, tag, recvbuf[i]);
            }
        }
    }
    ta_free(recvbuf);
    return 0;
}

References MPIU_Trace(), NTask, ta_free, ta_malloc, and ThisTask.

Here is the call graph for this function:

atomic_add_and_fetch()

static int atomic_add_and_fetch ( int *  ptr, int  value  )

inlinestatic

Definition at line 87 of file system.h.

                                                             {
    int k;
#pragma omp atomic capture
    {
      (*ptr)+=value;
      k = (*ptr);
    }
    return k;
}

atomic_fetch_and_add()

static int atomic_fetch_and_add ( int *  ptr, int  value  )

inlinestatic

Definition at line 78 of file system.h.

                                                             {
    int k;
#pragma omp atomic capture
    {
      k = (*ptr);
      (*ptr)+=value;
    }
    return k;
}

Referenced by get_freenode(), and real_ev().

Here is the caller graph for this function:

atomic_fetch_and_add_64()

static int64_t atomic_fetch_and_add_64 ( int64_t *  ptr, int64_t  value  )

inlinestatic

Definition at line 68 of file system.h.

                                                                            {
    int64_t k;
#pragma omp atomic capture
    {
      k = (*ptr);
      (*ptr)+=value;
    }
    return k;
}

Referenced by sfr_wind_feedback_ngbiter(), slots_convert(), slots_split_particle(), and timestep_eh_slots_fork().

Here is the caller graph for this function:

cluster_get_num_hosts()

int cluster_get_num_hosts

(

void

)

Definition at line 434 of file system.c.

{
    /* Find a unique hostid for the computing rank. */
    int NTask;
    int ThisTask;
    MPI_Comm_size(MPI_COMM_WORLD, &NTask);
    MPI_Comm_rank(MPI_COMM_WORLD, &ThisTask);
 
    /* Size is set by the size of the temp heap:
     * this fills it and should be changed if needed.*/
    const int bufsz = 256;
    char * buffer = ta_malloc("buffer", char, bufsz * NTask);
    memset(buffer, 0, bufsz * NTask);
    int i, j;
    gethostname(&buffer[bufsz*ThisTask], bufsz);
    buffer[bufsz * ThisTask + bufsz - 1] = '\0';
    MPI_Allgather(MPI_IN_PLACE, bufsz, MPI_CHAR, buffer, bufsz, MPI_CHAR, MPI_COMM_WORLD);
 
    int nunique = 0;
    /* Count unique entries*/
    for(j = 0; j < NTask; j++) {
        for(i = j+1; i < NTask; i++) {
            if(strncmp(buffer + i * bufsz, buffer + j * bufsz, bufsz) == 0)
                break;
        }
        if(i == NTask)
            nunique++;
    }
    ta_free(buffer);
    return nunique;
}

References NTask, ta_free, ta_malloc, and ThisTask.

Referenced by mymalloc_init().

Here is the caller graph for this function:

count_sum()

int64_t count_sum

(

int64_t

countLocal

)

Definition at line 264 of file system.c.

                                      {
    int64_t sum = 0;
    MPI_Allreduce(&countLocal, &sum, 1, MPI_INT64, MPI_SUM, MPI_COMM_WORLD);
    return sum;
}

References MPI_INT64.

Referenced by get_long_range_timestep_dloga(), and petaio_save_block().

Here is the caller graph for this function:

gadget_compact_thread_arrays()

size_t gadget_compact_thread_arrays	(	int *	dest,
		int *	srcs[],
		size_t	sizes[],
		int	narrays
	)

Definition at line 587 of file system.c.

{
    int i;
    size_t asize = 0;
 
    for(i = 0; i < narrays; i++)
    {
        memmove(dest + asize, srcs[i], sizeof(int) * sizes[i]);
        asize += sizes[i];
    }
    return asize;
}

Referenced by cooling_and_starformation(), domain_build_exchange_list(), rebuild_activelist(), treewalk_build_queue(), and treewalk_do_hsml_loop().

Here is the caller graph for this function:

gadget_setup_thread_arrays()

void gadget_setup_thread_arrays	(	int *	dest,
		int *	srcs[],
		size_t	sizes[],
		size_t	total_size,
		int	narrays
	)

Definition at line 600 of file system.c.

{
    int i;
    srcs[0] = dest;
    for(i=0; i < narrays; i++) {
        srcs[i] = dest + i * total_size;
        sizes[i] = 0;
    }
}

Referenced by cooling_and_starformation(), domain_build_exchange_list(), rebuild_activelist(), treewalk_build_queue(), and treewalk_do_hsml_loop().

Here is the caller graph for this function:

get_physmem_bytes()

double get_physmem_bytes

(

void

)

Definition at line 467 of file system.c.

{
#if defined _SC_PHYS_PAGES && defined _SC_PAGESIZE
    { /* This works on linux-gnu, solaris2 and cygwin.  */
        double pages = sysconf (_SC_PHYS_PAGES);
        double pagesize = sysconf (_SC_PAGESIZE);
        if (0 <= pages && 0 <= pagesize)
            return pages * pagesize;
    }
#endif
 
#if defined HW_PHYSMEM
    { /* This works on *bsd and darwin.  */
        unsigned int physmem;
        size_t len = sizeof physmem;
        static int mib[2] = { CTL_HW, HW_PHYSMEM };
 
        if (sysctl (mib, ARRAY_SIZE (mib), &physmem, &len, NULL, 0) == 0
                && len == sizeof (physmem))
            return (double) physmem;
    }
#endif
    return 64 * 1024 * 1024;
}

Referenced by read_parameter_file(), and read_parameterfile().

Here is the caller graph for this function:

get_random_number()

double get_random_number

(

uint64_t

id

)

Definition at line 60 of file system.c.

{
    return RndTable[(int)(id % RNDTABLE)];
}

References RNDTABLE, and RndTable.

Referenced by bh_powerlaw_seed_mass(), blackhole_accretion_ngbiter(), choose_QSO_halo(), gaussian_rng(), get_random_dir(), get_sfr_eeqos(), get_wind_dir(), lightcone_cross(), quicklyastarformation(), sfr_wind_feedback_ngbiter(), and update_random_offset().

Here is the caller graph for this function:

MPI_Alltoallv_smart()

int MPI_Alltoallv_smart	(	void *	sendbuf,
		int *	sendcnts,
		int *	sdispls,
		MPI_Datatype	sendtype,
		void *	recvbuf,
		int *	recvcnts,
		int *	rdispls,
		MPI_Datatype	recvtype,
		MPI_Comm	comm
	)

Definition at line 278 of file system.c.

{
    int ThisTask;
    int NTask;
    MPI_Comm_rank(comm, &ThisTask);
    MPI_Comm_size(comm, &NTask);
    int i;
    int nn = 0;
    int *a_sdispls=NULL, *a_recvcnts=NULL, *a_rdispls=NULL;
    for(i = 0; i < NTask; i ++) {
        if(sendcnts[i] > 0) {
            nn ++;
        }
    }
    if(recvcnts == NULL) {
        a_recvcnts = ta_malloc("recvcnts", int, NTask);
        recvcnts = a_recvcnts;
        MPI_Alltoall(sendcnts, 1, MPI_INT,
                     recvcnts, 1, MPI_INT, comm);
    }
    if(recvbuf == NULL) {
        int totalrecv = 0;
        for(i = 0; i < NTask; i ++) {
            totalrecv += recvcnts[i];
        }
        return totalrecv;
    }
    if(sdispls == NULL) {
        a_sdispls = ta_malloc("sdispls", int, NTask);
        sdispls = a_sdispls;
        sdispls[0] = 0;
        for (i = 1; i < NTask; i++) {
            sdispls[i] = sdispls[i - 1] + sendcnts[i - 1];
        }
    }
    if(rdispls == NULL) {
        a_rdispls = ta_malloc("rdispls", int, NTask);
        rdispls = a_rdispls;
        rdispls[0] = 0;
        for (i = 1; i < NTask; i++) {
            rdispls[i] = rdispls[i - 1] + recvcnts[i - 1];
        }
    }
 
    int dense = nn < NTask * 0.2;
    int tot_dense = 0, ret;
    MPI_Allreduce(&dense, &tot_dense, 1, MPI_INT, MPI_SUM, comm);
 
    if(tot_dense != 0) {
        ret = MPI_Alltoallv(sendbuf, sendcnts, sdispls,
                    sendtype, recvbuf,
                    recvcnts, rdispls, recvtype, comm);
    } else {
        ret = MPI_Alltoallv_sparse(sendbuf, sendcnts, sdispls,
                    sendtype, recvbuf,
                    recvcnts, rdispls, recvtype, comm);
 
    }
    if(a_rdispls)
        ta_free(a_rdispls);
    if(a_sdispls)
        ta_free(a_sdispls);
    if(a_recvcnts)
        ta_free(a_recvcnts);
    return ret;
}

References MPI_Alltoallv_sparse(), NTask, ta_free, ta_malloc, and ThisTask.

Referenced by fof_reduce_groups(), fof_seed(), and mpsort_mpi_histogram_sort().

Here is the call graph for this function:

Here is the caller graph for this function:

MPI_Alltoallv_sparse()

int MPI_Alltoallv_sparse	(	void *	sendbuf,
		int *	sendcnts,
		int *	sdispls,
		MPI_Datatype	sendtype,
		void *	recvbuf,
		int *	recvcnts,
		int *	rdispls,
		MPI_Datatype	recvtype,
		MPI_Comm	comm
	)

Definition at line 353 of file system.c.

                                                            {
 
    int ThisTask;
    int NTask;
    MPI_Comm_rank(comm, &ThisTask);
    MPI_Comm_size(comm, &NTask);
    int PTask;
    int ngrp;
 
    for(PTask = 0; NTask > (1 << PTask); PTask++);
 
    ptrdiff_t lb;
    ptrdiff_t send_elsize;
    ptrdiff_t recv_elsize;
 
    MPI_Type_get_extent(sendtype, &lb, &send_elsize);
    MPI_Type_get_extent(recvtype, &lb, &recv_elsize);
 
#ifndef NO_ISEND_IRECV_IN_DOMAIN
    int n_requests;
    MPI_Request *requests = (MPI_Request *) mymalloc("requests", NTask * 2 * sizeof(MPI_Request));
    n_requests = 0;
 
    for(ngrp = 0; ngrp < (1 << PTask); ngrp++)
    {
        int target = ThisTask ^ ngrp;
 
        if(target >= NTask) continue;
        if(recvcnts[target] == 0) continue;
        MPI_Irecv(
                ((char*) recvbuf) + recv_elsize * rdispls[target],
                recvcnts[target],
                recvtype, target, 101934, comm, &requests[n_requests++]);
    }
 
    MPI_Barrier(comm);
    /* not really necessary, but this will guarantee that all receives are
       posted before the sends, which helps the stability of MPI on
       bluegene, and perhaps some mpich1-clusters */
    /* Note 08/2016: Even on modern hardware this barrier leads to a slight speedup.
     * Probably because it allows the code to hit a fast path transfer.*/
 
    for(ngrp = 0; ngrp < (1 << PTask); ngrp++)
    {
        int target = ThisTask ^ ngrp;
        if(target >= NTask) continue;
        if(sendcnts[target] == 0) continue;
        MPI_Isend(((char*) sendbuf) + send_elsize * sdispls[target],
                sendcnts[target],
                sendtype, target, 101934, comm, &requests[n_requests++]);
    }
 
    MPI_Waitall(n_requests, requests, MPI_STATUSES_IGNORE);
    myfree(requests);
#else
    for(ngrp = 0; ngrp < (1 << PTask); ngrp++)
    {
        int target = ThisTask ^ ngrp;
 
        if(target >= NTask) continue;
        if(sendcnts[target] == 0 && recvcnts[target] == 0) continue;
        MPI_Sendrecv(((char*)sendbuf) + send_elsize * sdispls[target],
                sendcnts[target], sendtype,
                target, 101934,
                ((char*)recvbuf) + recv_elsize * rdispls[target],
                recvcnts[target], recvtype,
                target, 101934,
                comm, MPI_STATUS_IGNORE);
 
    }
#endif
    /* ensure the collective-ness */
    MPI_Barrier(comm);
 
    return 0;
}

References myfree, mymalloc, NTask, and ThisTask.

Referenced by domain_exchange_once(), ev_communicate(), and MPI_Alltoallv_smart().

Here is the caller graph for this function:

MPIU_Any()

int MPIU_Any	(	int	condition,
		MPI_Comm	comm
	)

Definition at line 545 of file system.c.

{
    MPI_Allreduce(MPI_IN_PLACE, &condition, 1, MPI_INT, MPI_LOR, comm);
    return condition;
}

Referenced by domain_decompose_full(), domain_determine_global_toptree(), domain_exchange(), domain_exchange_once(), domain_nonrecursively_combine_topTree(), force_tree_build(), mymalloc_init(), tamalloc_init(), winds_and_feedback(), and winds_subgrid().

Here is the caller graph for this function:

MPIU_Trace()

void MPIU_Trace	(	MPI_Comm	comm,
		int	where,
		const char *	fmt,
			...
	)

Definition at line 182 of file system.c.

{
    va_list va;
    va_start(va, fmt);
    MPIU_Tracev(comm, where, 0, fmt, va);
    va_end(va);
}

References MPIU_Tracev().

Referenced by _MPIU_Barrier().

Here is the call graph for this function:

Here is the caller graph for this function:

MPIU_Tracev()

void MPIU_Tracev	(	MPI_Comm	comm,
		int	where,
		int	error,
		const char *	fmt,
		va_list	va
	)

Definition at line 149 of file system.c.

{
    if(_timestart == -1) {
        _timestart = MPI_Wtime();
    }
    int ThisTask;
    MPI_Comm_rank(comm, &ThisTask);
    char prefix[128];
 
    char buf[4096];
    vsnprintf(buf, 4096, fmt, va);
    buf[4095] = '\0';
    char err[] = "ERROR: ";
    /* Print nothing if not an error*/
    if(!error)
        err[0] = '\0';
 
    if(where > 0) {
        sprintf(prefix, "[ %09.2f ] %sTask %d: ", MPI_Wtime() - _timestart, err, ThisTask);
    } else {
        sprintf(prefix, "[ %09.2f ] %s", MPI_Wtime() - _timestart, err);
    }
 
    if(ThisTask == 0 || where > 0) {
        putline(prefix, buf);
    }
}

References _timestart, prefix, putline(), and ThisTask.

Referenced by endrun(), message(), and MPIU_Trace().

Here is the call graph for this function:

Here is the caller graph for this function:

MPIU_write_pids()

void MPIU_write_pids

(

char *

filename

)

Definition at line 552 of file system.c.

{
    MPI_Comm comm = MPI_COMM_WORLD;
    int NTask;
    int ThisTask;
    MPI_Comm_size(comm, &NTask);
    MPI_Comm_rank(comm, &ThisTask);
 
    int my_pid = getpid();
    int * pids = ta_malloc("pids", int, NTask);
    /* Smaller buffer than in cluster_get_num_hosts because
     * here an overflow is harmless but running out of memory isn't*/
    int bufsz = 64;
    char * hosts = ta_malloc("hosts", char, (NTask+1) * bufsz);
    char * thishost = hosts + NTask * bufsz;
    gethostname(thishost, bufsz);
    thishost[bufsz - 1] = '\0';
    /* MPI_IN_PLACE is not used here because the MPI on travis doesn't like it*/
    MPI_Gather(thishost, bufsz, MPI_CHAR, hosts, bufsz, MPI_CHAR, 0, comm);
    MPI_Gather(&my_pid, 1, MPI_INT, pids, 1, MPI_INT, 0, comm);
 
    if(ThisTask == 0)
    {
        int i;
        FILE *fd = fopen(filename, "w");
        if(!fd)
            endrun(5, "Could not open pidfile %s\n", filename);
        for(i = 0; i < NTask; i++)
            fprintf(fd, "host: %s pid: %d\n", hosts+i*bufsz, pids[i]);
        fclose(fd);
    }
    myfree(hosts);
    myfree(pids);
}

References endrun(), myfree, NTask, ta_malloc, and ThisTask.

Referenced by begrun().

Here is the call graph for this function:

Here is the caller graph for this function:

second()

double second

(

void

)

Definition at line 83 of file system.c.

{
  return MPI_Wtime();
}

Referenced by ev_ndone(), ev_primary(), ev_reduce_result(), ev_secondary(), fof_label_primary(), and treewalk_run().

Here is the caller graph for this function:

set_random_numbers()

void set_random_numbers

(

int

seed

)

Definition at line 65 of file system.c.

{
    gsl_rng * random_generator = gsl_rng_alloc(gsl_rng_ranlxd1);
 
    /* start-up seed */
    gsl_rng_set(random_generator, seed);
 
    int i;
    for(i = 0; i < RNDTABLE; i++)
        RndTable[i] = gsl_rng_uniform(random_generator);
 
    gsl_rng_free(random_generator);
}

References RNDTABLE, and RndTable.

Referenced by begrun(), and run().

Here is the caller graph for this function:

sizemax()

size_t sizemax	(	size_t	a,
		size_t	b
	)

Definition at line 270 of file system.c.

{
  if(a < b)
    return b;
  else
    return a;
}

sumup_large_ints()

void sumup_large_ints	(	int	n,
		int *	src,
		int64_t *	res
	)

Definition at line 192 of file system.c.

{
    MPI_Comm comm = MPI_COMM_WORLD;
    int NTask;
    int ThisTask;
    MPI_Comm_size(comm, &NTask);
    MPI_Comm_rank(comm, &ThisTask);
 
    int i, j, *numlist;
 
    numlist = (int *) mymalloc("numlist", NTask * n * sizeof(int));
    MPI_Allgather(src, n, MPI_INT, numlist, n, MPI_INT, MPI_COMM_WORLD);
 
    for(j = 0; j < n; j++)
        res[j] = 0;
 
    for(i = 0; i < NTask; i++)
        for(j = 0; j < n; j++)
            res[j] += numlist[i * n + j];
 
    myfree(numlist);
}

References myfree, mymalloc, NTask, and ThisTask.

Referenced by check_omega(), collect_BH_info(), cooling_and_starformation(), fof_compile_catalogue(), fof_label_secondary(), gas_ionization_fraction(), get_long_range_timestep_dloga(), petaio_save_snapshot(), print_timebin_statistics(), turn_on_quasars(), winds_and_feedback(), and winds_find_weights().

Here is the caller graph for this function:

sumup_longs()

void sumup_longs	(	int	n,
		int64_t *	src,
		int64_t *	res
	)

Definition at line 215 of file system.c.

{
    MPI_Comm comm = MPI_COMM_WORLD;
    int NTask;
    int ThisTask;
    MPI_Comm_size(comm, &NTask);
    MPI_Comm_rank(comm, &ThisTask);
    int i, j;
    int64_t *numlist;
 
    numlist = (int64_t *) mymalloc("numlist", NTask * n * sizeof(int64_t));
    MPI_Allgather(src, n * sizeof(int64_t), MPI_BYTE, numlist, n * sizeof(int64_t), MPI_BYTE,
            MPI_COMM_WORLD);
 
    for(j = 0; j < n; j++)
        res[j] = 0;
 
    for(i = 0; i < NTask; i++)
        for(j = 0; j < n; j++)
            res[j] += numlist[i * n + j];
 
    myfree(numlist);
}

References myfree, mymalloc, NTask, and ThisTask.

timediff()

double timediff	(	double	t0,
		double	t1
	)

Definition at line 92 of file system.c.

{
  double dt;
 
  dt = t1 - t0;
 
  if(dt < 0)            /* overflow has occured (for systems with 32bit tick counter) */
    {
      dt = t1 + pow(2, 32) / CLOCKS_PER_SEC - t0;
    }
 
  return dt;
}

Referenced by ev_ndone(), ev_primary(), ev_reduce_result(), ev_secondary(), and treewalk_run().

Here is the caller graph for this function: