MP-Gadget  5.0.1.dev1-76bc7d4726-dirty
Classes | Macros | Functions
fofpetaio.c File Reference
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <bigfile-mpi.h>
#include "utils.h"
#include "utils/mpsort.h"
#include "partmanager.h"
#include "slotsmanager.h"
#include "petaio.h"
#include "exchange.h"
#include "fof.h"
#include "walltime.h"
Include dependency graph for fofpetaio.c:

Go to the source code of this file.

Classes

struct  PartIndex
 

Macros

#define SIMPLE_PROPERTY_FOF(name, field, type, items)
 

Functions

static void fof_register_io_blocks (int MetalReturnOn, int BlackHoleOn, struct IOTable *IOTable)
 
static void fof_write_header (BigFile *bf, int64_t TotNgroups, const double atime, const double *MassTable, Cosmology *CP, MPI_Comm Comm)
 
static void build_buffer_fof (FOFGroups *fof, BigArray *array, IOTableEntry *ent, struct conversions *conv)
 
static int fof_distribute_particles (struct part_manager_type *halo_pman, struct slots_manager_type *halo_sman, MPI_Comm Comm)
 
static void fof_radix_Group_GrNr (const void *a, void *radix, void *arg)
 
void fof_save_particles (FOFGroups *fof, const char *OutputDir, const char *FOFFileBase, int num, int SaveParticles, Cosmology *CP, double atime, const double *MassTable, int MetalReturnOn, int BlackholeOn, MPI_Comm Comm)
 
static int fof_sorted_layout (int i, const void *userdata)
 
static void fof_radix_sortkey (const void *c1, void *out, void *arg)
 
static void fof_radix_origin (const void *c1, void *out, void *arg)
 
static int order_by_type_and_grnr (const void *a, const void *b)
 
static int fof_try_particle_exchange (struct part_manager_type *halo_pman, struct slots_manager_type *halo_sman, MPI_Comm Comm)
 
static void GTFirstPos (int i, float *out, void *baseptr, void *smanptr, const struct conversions *params)
 
static void STFirstPos (int i, float *out, void *baseptr, void *smanptr, const struct conversions *params)
 
static void GTMassCenterPosition (int i, double *out, void *baseptr, void *smanptr, const struct conversions *params)
 
static void STMassCenterPosition (int i, double *out, void *baseptr, void *smanptr, const struct conversions *params)
 
static void GTMassCenterVelocity (int i, float *out, void *baseptr, void *slotptr, const struct conversions *params)
 

Macro Definition Documentation

◆ SIMPLE_PROPERTY_FOF

#define SIMPLE_PROPERTY_FOF (   name,
  field,
  type,
  items 
)
Value:
SIMPLE_GETTER(GT ## name , field, type, items, struct Group ) \
SIMPLE_SETTER(ST ## name , field, type, items, struct Group) \
const char * name
Definition: densitykernel.c:93
#define SIMPLE_GETTER(name, field, type, items, stype)
Definition: petaio.h:145
Definition: fof.h:26

Definition at line 394 of file fofpetaio.c.

Function Documentation

◆ build_buffer_fof()

static void build_buffer_fof ( FOFGroups fof,
BigArray *  array,
IOTableEntry ent,
struct conversions conv 
)
static

Definition at line 333 of file fofpetaio.c.

333  {
334 
335  int64_t npartLocal = fof->Ngroups;
336 
337  petaio_alloc_buffer(array, ent, npartLocal);
338  /* fill the buffer */
339  char * p = (char *) array->data;
340  int i;
341  for(i = 0; i < fof->Ngroups; i ++) {
342  ent->getter(i, p, fof->Group, NULL, conv);
343  p += array->strides[0];
344  }
345 }
void petaio_alloc_buffer(BigArray *array, IOTableEntry *ent, int64_t localsize)
Definition: petaio.c:491
struct Group * Group
Definition: fof.h:63
int Ngroups
Definition: fof.h:66
property_getter getter
Definition: petaio.h:56

References IOTableEntry::getter, FOFGroups::Group, FOFGroups::Ngroups, and petaio_alloc_buffer().

Referenced by fof_save_particles().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ fof_distribute_particles()

static int fof_distribute_particles ( struct part_manager_type halo_pman,
struct slots_manager_type halo_sman,
MPI_Comm  Comm 
)
static

Definition at line 249 of file fofpetaio.c.

250 {
251  int64_t i, NpigLocal = 0;
252  int64_t GrNrMax = -1; /* will mark particles that are not in any group */
253  int64_t GrNrMaxGlobal = 0;
254 
255  /* SlotsManager Needs initializing!*/
256  memcpy(halo_sman, SlotsManager, sizeof(struct slots_manager_type));
257 
258  halo_sman->Base = NULL;
259  for(i = 0; i < 6; i ++) {
260  halo_sman->info[i].size = 0;
261  halo_sman->info[i].maxsize = 0;
262  }
263 
264  int64_t atleast[6]={0};
265  /* Count how many particles we have: array reductions are an openmp 4.5 feature.*/
266 #if (defined _OPENMP) && (_OPENMP >= 201511)
267  #pragma omp parallel for reduction(+: NpigLocal, atleast)
268 #endif
269  for(i = 0; i < PartManager->NumPart; i ++) {
270  if(P[i].GrNr >= 0) {
271  NpigLocal++;
272  int type = P[i].Type;
273  /* How many of slot type?*/
274  if(type < 6 && type >= 0 && halo_sman->info[type].enabled)
275  atleast[type]++;
276  }
277  }
278  double FOFPartAllocFactor = (double) PartManager->MaxPart / PartManager->NumPart;
279  halo_pman->MaxPart = NpigLocal * FOFPartAllocFactor;
280  struct particle_data * halopart = (struct particle_data *) mymalloc("HaloParticle", sizeof(struct particle_data) * halo_pman->MaxPart);
281  halo_pman->Base = halopart;
282  halo_pman->NumPart = NpigLocal;
283  halo_pman->BoxSize = PartManager->BoxSize;
285 
286  /* We leave extra space in the hope that we can avoid compacting slots in the fof exchange*/
287  for(i = 0; i < 6; i ++)
288  atleast[i]*= FOFPartAllocFactor;
289 
290  slots_reserve(0, atleast, halo_sman);
291 
292  NpigLocal = 0;
293 
294  for(i = 0; i < PartManager->NumPart; i ++) {
295  if(P[i].GrNr < 0)
296  continue;
297  if(P[i].GrNr > GrNrMax)
298  GrNrMax = P[i].GrNr;
299  memcpy(&halo_pman->Base[NpigLocal], &P[i], sizeof(P[i]));
300  struct slot_info * info = &(halo_sman->info[P[i].Type]);
301  char * oldslotptr = SlotsManager->info[P[i].Type].ptr;
302  if(info->enabled) {
303  memcpy(info->ptr + info->size * info->elsize, oldslotptr+P[i].PI * info->elsize, info->elsize);
304  halo_pman->Base[NpigLocal].PI = info->size;
305  info->size++;
306  }
307  NpigLocal ++;
308  }
309  if(NpigLocal != halo_pman->NumPart)
310  endrun(3, "Error in NpigLocal %ld != %ld!\n", NpigLocal, halo_pman->NumPart);
311  MPI_Allreduce(&GrNrMax, &GrNrMaxGlobal, 1, MPI_INT, MPI_MAX, Comm);
312  message(0, "GrNrMax before exchange is %d\n", GrNrMaxGlobal);
313 
314  if(fof_try_particle_exchange(halo_pman, halo_sman, Comm)) {
315  message(1930, "Failed to exchange and write particles for the FOF. This is non-fatal, continuing\n");
316  return 1;
317  }
318 
319  /* Sort locally by group number*/
320  qsort_openmp(halopart, halo_pman->NumPart, sizeof(struct particle_data), order_by_type_and_grnr);
321  GrNrMax = -1;
322  #pragma omp parallel for reduction(max: GrNrMax)
323  for(i = 0; i < halo_pman->NumPart; i ++) {
324  if(halopart[i].GrNr > GrNrMax)
325  GrNrMax = halopart[i].GrNr;
326  }
327 
328  MPI_Allreduce(&GrNrMax, &GrNrMaxGlobal, 1, MPI_INT, MPI_MAX, Comm);
329  message(0, "GrNrMax after exchange is %d\n", GrNrMaxGlobal);
330  return 0;
331 }
void message(int where, const char *fmt,...)
Definition: endrun.c:175
void endrun(int where, const char *fmt,...)
Definition: endrun.c:147
static int order_by_type_and_grnr(const void *a, const void *b)
Definition: fofpetaio.c:160
static int fof_try_particle_exchange(struct part_manager_type *halo_pman, struct slots_manager_type *halo_sman, MPI_Comm Comm)
Definition: fofpetaio.c:178
#define mymalloc(name, size)
Definition: mymalloc.h:15
struct part_manager_type PartManager[1]
Definition: partmanager.c:11
#define P
Definition: partmanager.h:88
size_t slots_reserve(int where, int64_t atleast[6], struct slots_manager_type *sman)
Definition: slotsmanager.c:475
struct slots_manager_type SlotsManager[1]
Definition: slotsmanager.c:7
struct particle_data * Base
Definition: partmanager.h:74
double CurrentParticleOffset[3]
Definition: partmanager.h:82
size_t elsize
Definition: slotsmanager.h:13
int64_t maxsize
Definition: slotsmanager.h:11
int64_t size
Definition: slotsmanager.h:12
char * ptr
Definition: slotsmanager.h:10
struct slot_info info[6]
Definition: slotsmanager.h:112
#define qsort_openmp
Definition: test_exchange.c:14

References part_manager_type::Base, slots_manager_type::Base, part_manager_type::BoxSize, part_manager_type::CurrentParticleOffset, slot_info::elsize, slot_info::enabled, endrun(), fof_try_particle_exchange(), slots_manager_type::info, part_manager_type::MaxPart, slot_info::maxsize, message(), mymalloc, part_manager_type::NumPart, order_by_type_and_grnr(), P, PartManager, particle_data::PI, slot_info::ptr, qsort_openmp, slot_info::size, slots_reserve(), and SlotsManager.

Referenced by fof_save_particles().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ fof_radix_Group_GrNr()

static void fof_radix_Group_GrNr ( const void *  a,
void *  radix,
void *  arg 
)
static

Definition at line 26 of file fofpetaio.c.

26  {
27  uint64_t * u = (uint64_t *) radix;
28  struct BaseGroup * f = (struct BaseGroup*) a;
29  u[0] = f->GrNr;
30 }
Definition: fof.h:13
int GrNr
Definition: fof.h:17

References BaseGroup::GrNr.

Referenced by fof_save_particles().

Here is the caller graph for this function:

◆ fof_radix_origin()

static void fof_radix_origin ( const void *  c1,
void *  out,
void *  arg 
)
static

Definition at line 137 of file fofpetaio.c.

137  {
138  uint64_t * u = (uint64_t *) out;
139  const struct PartIndex * pi = (const struct PartIndex *) c1;
140  *u = pi->origin;
141 }
uint64_t origin
Definition: fofpetaio.c:120

References PartIndex::origin.

Referenced by fof_try_particle_exchange().

Here is the caller graph for this function:

◆ fof_radix_sortkey()

static void fof_radix_sortkey ( const void *  c1,
void *  out,
void *  arg 
)
static

Definition at line 132 of file fofpetaio.c.

132  {
133  uint64_t * u = (uint64_t *) out;
134  const struct PartIndex * pi = (const struct PartIndex *) c1;
135  *u = pi->sortKey;
136 }
int64_t sortKey
Definition: fofpetaio.c:122

References PartIndex::sortKey.

Referenced by fof_try_particle_exchange().

Here is the caller graph for this function:

◆ fof_register_io_blocks()

static void fof_register_io_blocks ( int  MetalReturnOn,
int  BlackHoleOn,
struct IOTable IOTable 
)
static

Definition at line 468 of file fofpetaio.c.

468  {
469  IOTable->used = 0;
470  IOTable->allocated = 100;
471  /* Allocate high so we can do a domain exchange,
472  * potentially increasing the slots, around this*/
473  IOTable->ent = (struct IOTableEntry *) mymalloc2("IOTable", IOTable->allocated* sizeof(IOTableEntry));
474 
475  IO_REG(GroupID, "u4", 1, PTYPE_FOF_GROUP, IOTable);
476  IO_REG(Mass, "f4", 1, PTYPE_FOF_GROUP, IOTable);
477  IO_REG(MassCenterPosition, "f8", 3, PTYPE_FOF_GROUP, IOTable);
478  IO_REG(FirstPos, "f4", 3, PTYPE_FOF_GROUP, IOTable);
479  IO_REG(MinID, "u8", 1, PTYPE_FOF_GROUP, IOTable);
480  IO_REG(Imom, "f4", 9, PTYPE_FOF_GROUP, IOTable);
481  IO_REG(Jmom, "f4", 3, PTYPE_FOF_GROUP, IOTable);
482  IO_REG_WRONLY(MassCenterVelocity, "f4", 3, PTYPE_FOF_GROUP, IOTable);
483  IO_REG(LengthByType, "u4", 6, PTYPE_FOF_GROUP, IOTable);
484  IO_REG(MassByType, "f4", 6, PTYPE_FOF_GROUP, IOTable);
485  IO_REG(MassHeIonized, "f4", 1, PTYPE_FOF_GROUP, IOTable);
486  /* Zero if star formation is not on*/
487  IO_REG(StarFormationRate, "f4", 1, PTYPE_FOF_GROUP, IOTable);
488  if(MetalReturnOn) {
489  IO_REG(GasMetalMass, "f4", 1, PTYPE_FOF_GROUP, IOTable);
490  IO_REG(StellarMetalMass, "f4", 1, PTYPE_FOF_GROUP, IOTable);
491  /* Zero if metal return is not on*/
492  IO_REG(GasMetalElemMass, "f4", NMETALS, PTYPE_FOF_GROUP, IOTable);
493  IO_REG(StellarMetalElemMass, "f4", NMETALS, PTYPE_FOF_GROUP, IOTable);
494  }
495  /* Zero if black hole is not on*/
496  if(BlackholeOn) {
497  IO_REG(BlackholeMass, "f4", 1, PTYPE_FOF_GROUP, IOTable);
498  IO_REG(BlackholeAccretionRate, "f4", 1, PTYPE_FOF_GROUP, IOTable);
499  }
500 }
#define mymalloc2(name, size)
Definition: mymalloc.h:16
#define IO_REG_WRONLY(name, dtype, items, ptype, IOTable)
Definition: petaio.h:117
#define PTYPE_FOF_GROUP
Definition: petaio.h:66
#define IO_REG(name, dtype, items, ptype, IOTable)
Definition: petaio.h:113
#define NMETALS
Definition: slotsmanager.h:60
Definition: petaio.h:60
int used
Definition: petaio.h:62
int allocated
Definition: petaio.h:63
IOTableEntry * ent
Definition: petaio.h:61

References IOTable::allocated, IOTable::ent, IO_REG, IO_REG_WRONLY, mymalloc2, NMETALS, PTYPE_FOF_GROUP, and IOTable::used.

Referenced by fof_save_particles().

Here is the caller graph for this function:

◆ fof_save_particles()

void fof_save_particles ( FOFGroups fof,
const char *  OutputDir,
const char *  FOFFileBase,
int  num,
int  SaveParticles,
Cosmology CP,
double  atime,
const double *  MassTable,
int  MetalReturnOn,
int  BlackholeOn,
MPI_Comm  Comm 
)

Definition at line 32 of file fofpetaio.c.

32  {
33  int i;
34  struct IOTable FOFIOTable = {0};
35  char * fname = fastpm_strdup_printf("%s/%s_%03d", OutputDir, FOFFileBase, num);
36  message(0, "Saving particle groups into %s\n", fname);
37 
38  fof_register_io_blocks(MetalReturnOn, BlackholeOn, &FOFIOTable);
39  /* sort the groups according to group-number */
40  mpsort_mpi(fof->Group, fof->Ngroups, sizeof(struct Group),
41  fof_radix_Group_GrNr, 8, NULL, Comm);
42 
43  BigFile bf = {0};
44  if(0 != big_file_mpi_create(&bf, fname, Comm)) {
45  endrun(0, "Failed to open file at %s\n", fname);
46  }
47  myfree(fname);
48  struct conversions conv = {0};
49  conv.atime = atime;
50  conv.hubble = hubble_function(CP, atime);
51 
52  MPIU_Barrier(Comm);
53  fof_write_header(&bf, fof->TotNgroups, atime, MassTable, CP, Comm);
54 
55  for(i = 0; i < FOFIOTable.used; i ++) {
56  /* only process the particle blocks */
57  char blockname[128];
58  int ptype = FOFIOTable.ent[i].ptype;
59  BigArray array = {0};
60  if(ptype == PTYPE_FOF_GROUP) {
61  sprintf(blockname, "FOFGroups/%s", FOFIOTable.ent[i].name);
62  build_buffer_fof(fof, &array, &FOFIOTable.ent[i], &conv);
63  message(0, "Writing Block %s\n", blockname);
64 
65  petaio_save_block(&bf, blockname, &array, 1);
66  petaio_destroy_buffer(&array);
67  }
68  }
69  destroy_io_blocks(&FOFIOTable);
70  walltime_measure("/FOF/IO/WriteFOF");
71 
72  if(SaveParticles) {
73  struct IOTable IOTable = {0};
74  register_io_blocks(&IOTable, 1, MetalReturnOn);
75  struct part_manager_type halo_pman = {0};
76  struct slots_manager_type halo_sman = {0};
77  if(fof_distribute_particles(&halo_pman, &halo_sman, Comm)) {
78  myfree(halo_sman.Base);
79  myfree(halo_pman.Base);
81  return;
82  }
83 
84  int * selection = (int *) mymalloc("Selection", sizeof(int) * halo_pman.NumPart);
85 
86  int ptype_offset[6]={0};
87  int ptype_count[6]={0};
88  petaio_build_selection(selection, ptype_offset, ptype_count, halo_pman.Base, halo_pman.NumPart, NULL);
89 
90  walltime_measure("/FOF/IO/argind");
91 
92  for(i = 0; i < IOTable.used; i ++) {
93  /* only process the particle blocks */
94  char blockname[128];
95  int ptype = IOTable.ent[i].ptype;
96  BigArray array = {0};
97  if(ptype < 6 && ptype >= 0) {
98  sprintf(blockname, "%d/%s", ptype, IOTable.ent[i].name);
99  petaio_build_buffer(&array, &IOTable.ent[i], selection + ptype_offset[ptype], ptype_count[ptype], halo_pman.Base, &halo_sman, &conv);
100 
101  message(0, "Writing Block %s\n", blockname);
102 
103  petaio_save_block(&bf, blockname, &array, 1);
104  petaio_destroy_buffer(&array);
105  }
106  }
107  myfree(selection);
108  myfree(halo_sman.Base);
109  myfree(halo_pman.Base);
110  walltime_measure("/FOF/IO/WriteParticles");
112  }
113 
114  big_file_mpi_close(&bf, Comm);
115 
116  message(0, "Group catalogues saved.\n");
117 }
double hubble_function(const Cosmology *CP, double a)
Definition: cosmology.c:58
static void build_buffer_fof(FOFGroups *fof, BigArray *array, IOTableEntry *ent, struct conversions *conv)
Definition: fofpetaio.c:333
static int fof_distribute_particles(struct part_manager_type *halo_pman, struct slots_manager_type *halo_sman, MPI_Comm Comm)
Definition: fofpetaio.c:249
static void fof_write_header(BigFile *bf, int64_t TotNgroups, const double atime, const double *MassTable, Cosmology *CP, MPI_Comm Comm)
Definition: fofpetaio.c:347
static void fof_register_io_blocks(int MetalReturnOn, int BlackHoleOn, struct IOTable *IOTable)
Definition: fofpetaio.c:468
static void fof_radix_Group_GrNr(const void *a, void *radix, void *arg)
Definition: fofpetaio.c:26
#define mpsort_mpi(base, nmemb, elsize, radix, rsize, arg, comm)
Definition: mpsort.h:26
#define myfree(x)
Definition: mymalloc.h:19
void petaio_destroy_buffer(BigArray *array)
Definition: petaio.c:557
void destroy_io_blocks(struct IOTable *IOTable)
Definition: petaio.c:1034
void register_io_blocks(struct IOTable *IOTable, int WriteGroupID, int MetalReturnOn)
Definition: petaio.c:909
void petaio_build_selection(int *selection, int *ptype_offset, int *ptype_count, const struct particle_data *Parts, const int NumPart, int(*select_func)(int i, const struct particle_data *Parts))
Definition: petaio.c:113
void petaio_build_buffer(BigArray *array, IOTableEntry *ent, const int *selection, const int NumSelection, struct particle_data *Parts, struct slots_manager_type *SlotsManager, struct conversions *conv)
Definition: petaio.c:521
void petaio_save_block(BigFile *bf, const char *blockname, BigArray *array, int verbose)
Definition: petaio.c:587
static Cosmology * CP
Definition: power.c:27
char * fastpm_strdup_printf(const char *fmt,...)
Definition: string.c:41
int64_t TotNgroups
Definition: fof.h:67
int ptype
Definition: petaio.h:52
char name[64]
Definition: petaio.h:51
double atime
Definition: petaio.h:41
double hubble
Definition: petaio.h:42
#define MPIU_Barrier(comm)
Definition: system.h:103
#define walltime_measure(name)
Definition: walltime.h:8
static enum TransferType ptype
Definition: zeldovich.c:146

References conversions::atime, part_manager_type::Base, slots_manager_type::Base, build_buffer_fof(), CP, destroy_io_blocks(), endrun(), IOTable::ent, fastpm_strdup_printf(), fof_distribute_particles(), fof_radix_Group_GrNr(), fof_register_io_blocks(), fof_write_header(), FOFGroups::Group, conversions::hubble, hubble_function(), message(), MPIU_Barrier, mpsort_mpi, myfree, mymalloc, IOTableEntry::name, FOFGroups::Ngroups, part_manager_type::NumPart, petaio_build_buffer(), petaio_build_selection(), petaio_destroy_buffer(), petaio_save_block(), IOTableEntry::ptype, ptype, PTYPE_FOF_GROUP, register_io_blocks(), FOFGroups::TotNgroups, IOTable::used, and walltime_measure.

Referenced by fof_save_groups().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ fof_sorted_layout()

static int fof_sorted_layout ( int  i,
const void *  userdata 
)
static

Definition at line 127 of file fofpetaio.c.

127  {
128  struct part_manager_type * halo_pman = (struct part_manager_type *) userdata;
129  return halo_pman->Base[i].TargetTask;
130 }

References part_manager_type::Base, and particle_data::TargetTask.

Referenced by fof_try_particle_exchange().

Here is the caller graph for this function:

◆ fof_try_particle_exchange()

static int fof_try_particle_exchange ( struct part_manager_type halo_pman,
struct slots_manager_type halo_sman,
MPI_Comm  Comm 
)
static

Definition at line 178 of file fofpetaio.c.

179 {
180  struct PartIndex * pi = (struct PartIndex *) mymalloc("PartIndex", sizeof(struct PartIndex) * halo_pman->NumPart);
181  int ThisTask;
182  MPI_Comm_rank(Comm, &ThisTask);
183 
184  int64_t i = 0;
185  /* Build the index: needs to be done each time we loop as may have changed*/
186  /* Yu: found it! this shall be int64 */
187  const uint64_t task_origin_offset = PartManager->MaxPart + 1Lu;
188  #pragma omp parallel for
189  for(i = 0; i < halo_pman->NumPart; i ++) {
190  pi[i].origin = task_origin_offset * ((uint64_t) ThisTask) + i;
191  pi[i].sortKey = halo_pman->Base[i].GrNr;
192  }
193  /* sort pi to decide targetTask */
194  mpsort_mpi(pi, halo_pman->NumPart, sizeof(struct PartIndex),
195  fof_radix_sortkey, 8, NULL, Comm);
196 
197  //int64_t Npig = count_sum(NpigLocal);
198  //int64_t offsetLocal = MPIU_cumsum(NpigLocal, Comm);
199 
200  //size_t chunksize = (Npig / NTask) + (Npig % NTask != 0);
201 
202  #pragma omp parallel for
203  for(i = 0; i < halo_pman->NumPart; i ++) {
204 /* YU: A typo error here, should be IMIN, DMIN is for double but this should have tainted TargetTask,
205  offset and chunksize are int */
206  //ptrdiff_t offset = offsetLocal + i;
207  //pi[i].targetTask = IMIN(offset / chunksize, NTask - 1);
208  /* YU: let's see if we keep the FOF particle load on the processes, IO would be faster
209  (as at high z many ranks has no FOF), communication becomes sparse. */
210  pi[i].targetTask = ThisTask;
211  }
212  /* return pi to the original processors */
213  mpsort_mpi(pi, halo_pman->NumPart, sizeof(struct PartIndex), fof_radix_origin, 8, NULL, Comm);
214  /* Target task is copied into the particle table, unioned with Dthsml.
215  * This is a bit of a hack: probably the elegant thing to do is to unify slot
216  * and main structure, then mpsort the combination directly. */
217 #ifdef DEBUG
218  int NTask;
219  MPI_Comm_size(Comm, &NTask);
220  #pragma omp parallel for
221  for(i = 0; i < halo_pman->NumPart; i ++) {
222  halo_pman->Base[i].TargetTask = -1;
223  if(pi[i].targetTask >= NTask || pi[i].targetTask < 0)
224  endrun(23, "pi %d is impossible %d of %d tasks\n",i,pi[i].targetTask, NTask);
225  }
226 #endif
227  #pragma omp parallel for
228  for(i = 0; i < halo_pman->NumPart; i ++) {
229  size_t index = pi[i].origin % task_origin_offset;
230  if(index >= (size_t) halo_pman->NumPart)
231  endrun(23, "entry %d has index %lu (npiglocal %d)\n", i, index, halo_pman->NumPart);
232  halo_pman->Base[index].TargetTask = pi[i].targetTask;
233  }
234  myfree(pi);
235 #ifdef DEBUG
236  #pragma omp parallel for
237  for(i = 0; i < halo_pman->NumPart; i ++) {
238  if(halo_pman->Base[i].TargetTask < 0)
239  endrun(4, "TargetTask %d not changed %d! neighbours: %d %d\n", i, halo_pman->Base[i].TargetTask, halo_pman->Base[i-1].TargetTask, halo_pman->Base[i+1].TargetTask);
240  }
241 #endif
242 
243  walltime_measure("/FOF/IO/Distribute");
244 
245  return domain_exchange(fof_sorted_layout, halo_pman, 1, NULL, halo_pman, halo_sman, 10000, Comm);
246 }
int domain_exchange(ExchangeLayoutFunc layoutfunc, const void *layout_userdata, int do_gc, struct DriftData *drift, struct part_manager_type *pman, struct slots_manager_type *sman, int maxiter, MPI_Comm Comm)
Definition: exchange.c:103
static void fof_radix_sortkey(const void *c1, void *out, void *arg)
Definition: fofpetaio.c:132
static int fof_sorted_layout(int i, const void *userdata)
Definition: fofpetaio.c:127
static void fof_radix_origin(const void *c1, void *out, void *arg)
Definition: fofpetaio.c:137
int targetTask
Definition: fofpetaio.c:123
int64_t GrNr
Definition: partmanager.h:68
int ThisTask
Definition: test_exchange.c:23
int NTask
Definition: test_exchange.c:23

References part_manager_type::Base, domain_exchange(), endrun(), fof_radix_origin(), fof_radix_sortkey(), fof_sorted_layout(), particle_data::GrNr, part_manager_type::MaxPart, mpsort_mpi, myfree, mymalloc, NTask, part_manager_type::NumPart, PartIndex::origin, PartManager, PartIndex::sortKey, PartIndex::targetTask, particle_data::TargetTask, ThisTask, and walltime_measure.

Referenced by fof_distribute_particles().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ fof_write_header()

static void fof_write_header ( BigFile *  bf,
int64_t  TotNgroups,
const double  atime,
const double *  MassTable,
Cosmology CP,
MPI_Comm  Comm 
)
static

Definition at line 347 of file fofpetaio.c.

347  {
348  BigBlock bh;
349  if(0 != big_file_mpi_create_block(bf, &bh, "Header", NULL, 0, 0, 0, Comm)) {
350  endrun(0, "Failed to create header\n");
351  }
352  int i;
353  int k;
354  int64_t npartLocal[6];
355  int64_t npartTotal[6];
356 
357  for (k = 0; k < 6; k ++) {
358  npartLocal[k] = 0;
359  }
360 #if (defined _OPENMP) && (_OPENMP >= 201511)
361  #pragma omp parallel for reduction(+: npartLocal)
362 #endif
363  for (i = 0; i < PartManager->NumPart; i ++) {
364  if(P[i].GrNr < 0) continue; /* skip those not in groups */
365  npartLocal[P[i].Type] ++;
366  }
367 
368  MPI_Allreduce(npartLocal, npartTotal, 6, MPI_INT64, MPI_SUM, Comm);
369 
370  /* conversion from peculiar velocity to RSD */
371  const double hubble = hubble_function(CP, atime);
372  double RSD = 1.0 / (atime * hubble);
373 
374  int pecvel = GetUsePeculiarVelocity();
375  if(!pecvel) {
376  RSD /= atime; /* Conversion from internal velocity to RSD */
377  }
378  big_block_set_attr(&bh, "NumPartInGroupTotal", npartTotal, "u8", 6);
379  big_block_set_attr(&bh, "NumFOFGroupsTotal", &TotNgroups, "u8", 1);
380  big_block_set_attr(&bh, "RSDFactor", &RSD, "f8", 1);
381  big_block_set_attr(&bh, "MassTable", MassTable, "f8", 6);
382  big_block_set_attr(&bh, "Time", &atime, "f8", 1);
383  big_block_set_attr(&bh, "BoxSize", &PartManager->BoxSize, "f8", 1);
384  big_block_set_attr(&bh, "OmegaLambda", &CP->OmegaLambda, "f8", 1);
385  big_block_set_attr(&bh, "Omega0", &CP->Omega0, "f8", 1);
386  big_block_set_attr(&bh, "HubbleParam", &CP->HubbleParam, "f8", 1);
387  big_block_set_attr(&bh, "CMBTemperature", &CP->CMBTemperature, "f8", 1);
388  big_block_set_attr(&bh, "OmegaBaryon", &CP->OmegaBaryon, "f8", 1);
389  big_block_set_attr(&bh, "UsePeculiarVelocity", &pecvel, "i4", 1);
390  big_block_mpi_close(&bh, Comm);
391 }
int GetUsePeculiarVelocity(void)
Definition: petaio.c:79
double Omega0
Definition: cosmology.h:10
double HubbleParam
Definition: cosmology.h:20
double CMBTemperature
Definition: cosmology.h:9
double OmegaLambda
Definition: cosmology.h:14
double OmegaBaryon
Definition: cosmology.h:19
#define MPI_INT64
Definition: system.h:12

References part_manager_type::BoxSize, Cosmology::CMBTemperature, CP, endrun(), GetUsePeculiarVelocity(), hubble_function(), Cosmology::HubbleParam, MPI_INT64, part_manager_type::NumPart, Cosmology::Omega0, Cosmology::OmegaBaryon, Cosmology::OmegaLambda, P, and PartManager.

Referenced by fof_save_particles().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ GTFirstPos()

static void GTFirstPos ( int  i,
float *  out,
void *  baseptr,
void *  smanptr,
const struct conversions params 
)
static

Definition at line 404 of file fofpetaio.c.

404  {
405  /* Remove the particle offset before saving*/
406  struct Group * grp = (struct Group *) baseptr;
407  int d;
408  for(d = 0; d < 3; d ++) {
409  out[d] = grp[i].base.FirstPos[d] - PartManager->CurrentParticleOffset[d];
410  while(out[d] > PartManager->BoxSize) out[d] -= PartManager->BoxSize;
411  while(out[d] <= 0) out[d] += PartManager->BoxSize;
412  }
413 }
float FirstPos[3]
Definition: fof.h:22
struct BaseGroup base
Definition: fof.h:27

References Group::base, part_manager_type::BoxSize, part_manager_type::CurrentParticleOffset, BaseGroup::FirstPos, and PartManager.

◆ GTMassCenterPosition()

static void GTMassCenterPosition ( int  i,
double *  out,
void *  baseptr,
void *  smanptr,
const struct conversions params 
)
static

Definition at line 423 of file fofpetaio.c.

423  {
424  /* Remove the particle offset before saving*/
425  struct Group * grp = (struct Group *) baseptr;
426  int d;
427  for(d = 0; d < 3; d ++) {
428  out[d] = grp[i].CM[d] - PartManager->CurrentParticleOffset[d];
429  while(out[d] > PartManager->BoxSize) out[d] -= PartManager->BoxSize;
430  while(out[d] <= 0) out[d] += PartManager->BoxSize;
431  }
432 }
double CM[3]
Definition: fof.h:34

References part_manager_type::BoxSize, Group::CM, part_manager_type::CurrentParticleOffset, and PartManager.

◆ GTMassCenterVelocity()

static void GTMassCenterVelocity ( int  i,
float *  out,
void *  baseptr,
void *  slotptr,
const struct conversions params 
)
static

Definition at line 442 of file fofpetaio.c.

442  {
443  double fac;
444  struct Group * Group = (struct Group *) baseptr;
445  if (GetUsePeculiarVelocity()) {
446  fac = 1.0 / params->atime;
447  } else {
448  fac = 1.0;
449  }
450 
451  int d;
452  for(d = 0; d < 3; d ++) {
453  out[d] = fac * Group[i].Vel[d];
454  }
455 }
double Vel[3]
Definition: fof.h:35

References conversions::atime, GetUsePeculiarVelocity(), and Group::Vel.

Here is the call graph for this function:

◆ order_by_type_and_grnr()

static int order_by_type_and_grnr ( const void *  a,
const void *  b 
)
static

Definition at line 160 of file fofpetaio.c.

161 {
162  const struct particle_data * pa = (const struct particle_data *) a;
163  const struct particle_data * pb = (const struct particle_data *) b;
164 
165  if(pa->Type < pb->Type)
166  return -1;
167  if(pa->Type > pb->Type)
168  return +1;
169  if(pa->GrNr < pb->GrNr)
170  return -1;
171  if(pa->GrNr > pb->GrNr)
172  return +1;
173 
174  return 0;
175 }
unsigned int Type
Definition: partmanager.h:17

References particle_data::GrNr, and particle_data::Type.

Referenced by fof_distribute_particles().

Here is the caller graph for this function:

◆ STFirstPos()

static void STFirstPos ( int  i,
float *  out,
void *  baseptr,
void *  smanptr,
const struct conversions params 
)
static

Definition at line 415 of file fofpetaio.c.

415  {
416  int d;
417  struct Group * grp = (struct Group *) baseptr;
418  for(d = 0; d < 3; d ++) {
419  grp->base.FirstPos[i] = out[d];
420  }
421 }

References Group::base, and BaseGroup::FirstPos.

◆ STMassCenterPosition()

static void STMassCenterPosition ( int  i,
double *  out,
void *  baseptr,
void *  smanptr,
const struct conversions params 
)
static

Definition at line 434 of file fofpetaio.c.

434  {
435  int d;
436  struct Group * grp = (struct Group *) baseptr;
437  for(d = 0; d < 3; d ++) {
438  grp->CM[d] = out[d];
439  }
440 }

References Group::CM.