domain.h File Reference

#include <mpi.h>
#include "utils/peano.h"
#include "utils/paramset.h"
#include "drift.h"

Include dependency graph for domain.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
struct	topnode_data
struct	topleaf_data
struct	task_data
struct	DomainDecomp
struct	DomainParams

Typedefs
typedef struct DomainDecomp	DomainDecomp
typedef struct DomainParams	DomainParams

Functions
void	set_domain_params (ParameterSet *ps)
void	set_domain_par (DomainParams dp)
void	domain_decompose_full (DomainDecomp *ddecomp)
void	domain_maintain (DomainDecomp *ddecomp, struct DriftData *drift)
static int	domain_get_topleaf (const peano_t key, const DomainDecomp *ddecomp)
void	domain_free (DomainDecomp *ddecomp)

Typedef Documentation

DomainDecomp

typedef struct DomainDecomp DomainDecomp

DomainParams

typedef struct DomainParams DomainParams

Function Documentation

domain_decompose_full()

void domain_decompose_full

(

DomainDecomp *

ddecomp

)

This is the main routine for the domain decomposition. It acts as a driver routine that allocates various temporary buffers, maps the particles back onto the periodic box if needed, and then does the domain decomposition, and a final Peano-Hilbert order of all particles as a tuning measure.

Definition at line 155 of file domain.c.

{
    static DomainDecompositionPolicy policies[16];
    static int Npolicies = 0;
 
    /* start from last successful policy to avoid retries */
    static int LastSuccessfulPolicy = 0;
 
    if (Npolicies == 0) {
        const int NincreaseAlloc = 16;
        Npolicies = domain_policies_init(policies, NincreaseAlloc, 4);
    }
 
    walltime_measure("/Misc");
 
    message(0, "domain decomposition... (presently allocated=%g MB)\n", mymalloc_usedbytes() / (1024.0 * 1024.0));
 
    int decompose_failed = 1;
    int i;
    for(i = LastSuccessfulPolicy; i < Npolicies; i ++)
    {
        domain_free(ddecomp);
 
#ifdef DEBUG
        domain_test_id_uniqueness(PartManager);
#endif
        int MaxTopNodes = domain_allocate(ddecomp, &policies[i]);
 
        message(0, "Attempting new domain decomposition policy: TopNodeAllocFactor=%g, UseglobalSort=%d, SubSampleDistance=%d UsePreSort=%d\n",
                policies[i].TopNodeAllocFactor, policies[i].UseGlobalSort, policies[i].SubSampleDistance, policies[i].PreSort);
 
        decompose_failed = domain_attempt_decompose(ddecomp, &policies[i], MaxTopNodes);
        decompose_failed = MPIU_Any(decompose_failed, ddecomp->DomainComm);
 
        if(decompose_failed)
            continue;
 
        LastSuccessfulPolicy = i;
        if(domain_balance(ddecomp) == 0)
            break;
    }
 
    if(decompose_failed) {
        endrun(0, "No suitable domain decomposition policy worked for this particle distribution\n");
    }
 
    /* copy the used nodes from temp to the true. */
    struct topleaf_data * OldTopLeaves = ddecomp->TopLeaves;
    struct topnode_data * OldTopNodes = ddecomp->TopNodes;
 
    ddecomp->TopNodes  = (struct topnode_data *) mymalloc2("TopNodes", sizeof(ddecomp->TopNodes[0]) * ddecomp->NTopNodes);
    /* add 1 extra to mark the end of TopLeaves; see assign */
    ddecomp->TopLeaves = (struct topleaf_data *) mymalloc2("TopLeaves", sizeof(ddecomp->TopLeaves[0]) * (ddecomp->NTopLeaves + 1));
 
    memcpy(ddecomp->TopLeaves, OldTopLeaves, ddecomp->NTopLeaves* sizeof(ddecomp->TopLeaves[0]));
    memcpy(ddecomp->TopNodes, OldTopNodes, ddecomp->NTopNodes * sizeof(ddecomp->TopNodes[0]));
 
    /* no longer useful */
    myfree(OldTopLeaves);
    myfree(OldTopNodes);
 
    if(domain_exchange(domain_layoutfunc, ddecomp, 0, NULL, PartManager, SlotsManager, 10000, ddecomp->DomainComm))
        endrun(1929,"Could not exchange particles\n");
 
    /*Do a garbage collection so that the slots are ordered
     *the same as the particles, garbage is at the end and all particles are in peano order.*/
    slots_gc_sorted(PartManager, SlotsManager);
 
    /*Ensure collective*/
    MPIU_Barrier(ddecomp->DomainComm);
    message(0, "Domain decomposition done.\n");
 
    report_memory_usage("DOMAIN");
 
    walltime_measure("/Domain/PeanoSort");
}

References domain_allocate(), domain_attempt_decompose(), domain_balance(), domain_exchange(), domain_free(), domain_layoutfunc(), domain_policies_init(), domain_test_id_uniqueness(), DomainDecomp::DomainComm, endrun(), message(), MPIU_Any(), MPIU_Barrier, myfree, mymalloc2, mymalloc_usedbytes, DomainDecomp::NTopLeaves, DomainDecomp::NTopNodes, PartManager, report_memory_usage, slots_gc_sorted(), SlotsManager, DomainDecomp::TopLeaves, DomainDecomp::TopNodes, and walltime_measure.

Referenced by begrun(), do_force_test(), domain_maintain(), run(), run_gravity_test(), runfof(), runpower(), and test_fof().

Here is the call graph for this function:

Here is the caller graph for this function:

domain_free()

void domain_free

(

DomainDecomp *

ddecomp

)

Definition at line 320 of file domain.c.

{
    if(ddecomp->domain_allocated_flag)
    {
        myfree(ddecomp->TopLeaves);
        myfree(ddecomp->TopNodes);
        myfree(ddecomp->Tasks);
        ddecomp->domain_allocated_flag = 0;
    }
}

References DomainDecomp::domain_allocated_flag, myfree, DomainDecomp::Tasks, DomainDecomp::TopLeaves, and DomainDecomp::TopNodes.

Referenced by begrun(), do_force_test(), domain_decompose_full(), and run_gravity_test().

Here is the caller graph for this function:

domain_get_topleaf()

static int domain_get_topleaf ( const peano_t  key, const DomainDecomp *  ddecomp  )

inlinestatic

This function determines the TopLeaves entry for the given key.

Definition at line 74 of file domain.h.

                                                                    {
    int no=0;
    while(ddecomp->TopNodes[no].Daughter >= 0)
        no = ddecomp->TopNodes[no].Daughter + ((key - ddecomp->TopNodes[no].StartKey) >> (ddecomp->TopNodes[no].Shift - 3));
    no = ddecomp->TopNodes[no].Leaf;
    return no;
};

References topnode_data::Daughter, topnode_data::Leaf, topnode_data::Shift, topnode_data::StartKey, and DomainDecomp::TopNodes.

Referenced by domain_compute_costs(), domain_layoutfunc(), and force_tree_create_nodes().

Here is the caller graph for this function:

domain_maintain()

void domain_maintain	(	DomainDecomp *	ddecomp,
		struct DriftData *	drift
	)

Definition at line 234 of file domain.c.

{
    message(0, "Attempting a domain exchange\n");
 
    walltime_measure("/Misc");
 
    /* Try a domain exchange.
     * If we have no memory for the particles,
     * bail and do a full domain*/
    if(0 != domain_exchange(domain_layoutfunc, ddecomp, 0, drift, PartManager, SlotsManager, 10000, ddecomp->DomainComm)) {
        domain_decompose_full(ddecomp);
        return;
    }
}

References domain_decompose_full(), domain_exchange(), domain_layoutfunc(), DomainDecomp::DomainComm, message(), PartManager, SlotsManager, and walltime_measure.

Referenced by run().

Here is the call graph for this function:

Here is the caller graph for this function:

set_domain_par()

void set_domain_par

(

DomainParams

dp

)

Definition at line 78 of file domain.c.

{
    domain_params = dp;
}

References domain_params.

Referenced by setup_tree(), and test_fof().

Here is the caller graph for this function:

set_domain_params()

void set_domain_params

(

ParameterSet *

ps

)

Definition at line 84 of file domain.c.

{
    int ThisTask;
    MPI_Comm_rank(MPI_COMM_WORLD, &ThisTask);
    if(ThisTask == 0) {
        domain_params.DomainOverDecompositionFactor = param_get_int(ps, "DomainOverDecompositionFactor");
        /* Create one domain per thread. This helps the balance and makes the treebuild merge faster*/
        if(domain_params.DomainOverDecompositionFactor < 0)
            domain_params.DomainOverDecompositionFactor = omp_get_max_threads();
        if(domain_params.DomainOverDecompositionFactor < 4)
            domain_params.DomainOverDecompositionFactor = 4;
        domain_params.TopNodeAllocFactor = param_get_double(ps, "TopNodeAllocFactor");
        domain_params.DomainUseGlobalSorting = param_get_int(ps, "DomainUseGlobalSorting");
        domain_params.SetAsideFactor = 1.;
        if((param_get_int(ps, "StarformationOn") && param_get_double(ps, "QuickLymanAlphaProbability") == 0.)
            || param_get_int(ps, "BlackHoleOn"))
            domain_params.SetAsideFactor = 0.95;
    }
    MPI_Bcast(&domain_params, sizeof(DomainParams), MPI_BYTE, 0, MPI_COMM_WORLD);
}

References domain_params, DomainParams::DomainOverDecompositionFactor, DomainParams::DomainUseGlobalSorting, param_get_double(), param_get_int(), DomainParams::SetAsideFactor, ThisTask, and DomainParams::TopNodeAllocFactor.

Referenced by read_parameter_file().

Here is the call graph for this function:

Here is the caller graph for this function: