isl_map.c: improve documentation of isl_map_preimage_domain_pw_multi_aff
[isl.git] / isl_schedule.c
blob95555ad4a6a166fc110dc86ac3954a00a3d4d842
1 /*
2 * Copyright 2011 INRIA Saclay
3 * Copyright 2012-2013 Ecole Normale Superieure
5 * Use of this software is governed by the MIT license
7 * Written by Sven Verdoolaege, INRIA Saclay - Ile-de-France,
8 * Parc Club Orsay Universite, ZAC des vignes, 4 rue Jacques Monod,
9 * 91893 Orsay, France
10 * and Ecole Normale Superieure, 45 rue d'Ulm, 75230 Paris, France
13 #include <isl_ctx_private.h>
14 #include <isl_map_private.h>
15 #include <isl_space_private.h>
16 #include <isl_aff_private.h>
17 #include <isl/hash.h>
18 #include <isl/constraint.h>
19 #include <isl/schedule.h>
20 #include <isl_mat_private.h>
21 #include <isl_vec_private.h>
22 #include <isl/set.h>
23 #include <isl_seq.h>
24 #include <isl_tab.h>
25 #include <isl_dim_map.h>
26 #include <isl/map_to_basic_set.h>
27 #include <isl_sort.h>
28 #include <isl_schedule_private.h>
29 #include <isl_band_private.h>
30 #include <isl_options_private.h>
31 #include <isl_tarjan.h>
34 * The scheduling algorithm implemented in this file was inspired by
35 * Bondhugula et al., "Automatic Transformations for Communication-Minimized
36 * Parallelization and Locality Optimization in the Polyhedral Model".
39 __isl_give isl_schedule_constraints *isl_schedule_constraints_copy(
40 __isl_keep isl_schedule_constraints *sc)
42 isl_ctx *ctx;
43 isl_schedule_constraints *sc_copy;
44 enum isl_edge_type i;
46 ctx = isl_union_set_get_ctx(sc->domain);
47 sc_copy = isl_calloc_type(ctx, struct isl_schedule_constraints);
48 if (!sc_copy)
49 return NULL;
51 sc_copy->domain = isl_union_set_copy(sc->domain);
52 if (!sc_copy->domain)
53 return isl_schedule_constraints_free(sc_copy);
55 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
56 sc_copy->constraint[i] = isl_union_map_copy(sc->constraint[i]);
57 if (!sc_copy->constraint[i])
58 return isl_schedule_constraints_free(sc_copy);
61 return sc_copy;
65 /* Construct an isl_schedule_constraints object for computing a schedule
66 * on "domain". The initial object does not impose any constraints.
68 __isl_give isl_schedule_constraints *isl_schedule_constraints_on_domain(
69 __isl_take isl_union_set *domain)
71 isl_ctx *ctx;
72 isl_space *space;
73 isl_schedule_constraints *sc;
74 isl_union_map *empty;
75 enum isl_edge_type i;
77 if (!domain)
78 return NULL;
80 ctx = isl_union_set_get_ctx(domain);
81 sc = isl_calloc_type(ctx, struct isl_schedule_constraints);
82 if (!sc)
83 return isl_union_set_free(domain);
85 space = isl_union_set_get_space(domain);
86 sc->domain = domain;
87 empty = isl_union_map_empty(space);
88 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
89 sc->constraint[i] = isl_union_map_copy(empty);
90 if (!sc->constraint[i])
91 sc->domain = isl_union_set_free(sc->domain);
93 isl_union_map_free(empty);
95 if (!sc->domain)
96 return isl_schedule_constraints_free(sc);
98 return sc;
101 /* Replace the validity constraints of "sc" by "validity".
103 __isl_give isl_schedule_constraints *isl_schedule_constraints_set_validity(
104 __isl_take isl_schedule_constraints *sc,
105 __isl_take isl_union_map *validity)
107 if (!sc || !validity)
108 goto error;
110 isl_union_map_free(sc->constraint[isl_edge_validity]);
111 sc->constraint[isl_edge_validity] = validity;
113 return sc;
114 error:
115 isl_schedule_constraints_free(sc);
116 isl_union_map_free(validity);
117 return NULL;
120 /* Replace the coincidence constraints of "sc" by "coincidence".
122 __isl_give isl_schedule_constraints *isl_schedule_constraints_set_coincidence(
123 __isl_take isl_schedule_constraints *sc,
124 __isl_take isl_union_map *coincidence)
126 if (!sc || !coincidence)
127 goto error;
129 isl_union_map_free(sc->constraint[isl_edge_coincidence]);
130 sc->constraint[isl_edge_coincidence] = coincidence;
132 return sc;
133 error:
134 isl_schedule_constraints_free(sc);
135 isl_union_map_free(coincidence);
136 return NULL;
139 /* Replace the proximity constraints of "sc" by "proximity".
141 __isl_give isl_schedule_constraints *isl_schedule_constraints_set_proximity(
142 __isl_take isl_schedule_constraints *sc,
143 __isl_take isl_union_map *proximity)
145 if (!sc || !proximity)
146 goto error;
148 isl_union_map_free(sc->constraint[isl_edge_proximity]);
149 sc->constraint[isl_edge_proximity] = proximity;
151 return sc;
152 error:
153 isl_schedule_constraints_free(sc);
154 isl_union_map_free(proximity);
155 return NULL;
158 /* Replace the conditional validity constraints of "sc" by "condition"
159 * and "validity".
161 __isl_give isl_schedule_constraints *
162 isl_schedule_constraints_set_conditional_validity(
163 __isl_take isl_schedule_constraints *sc,
164 __isl_take isl_union_map *condition,
165 __isl_take isl_union_map *validity)
167 if (!sc || !condition || !validity)
168 goto error;
170 isl_union_map_free(sc->constraint[isl_edge_condition]);
171 sc->constraint[isl_edge_condition] = condition;
172 isl_union_map_free(sc->constraint[isl_edge_conditional_validity]);
173 sc->constraint[isl_edge_conditional_validity] = validity;
175 return sc;
176 error:
177 isl_schedule_constraints_free(sc);
178 isl_union_map_free(condition);
179 isl_union_map_free(validity);
180 return NULL;
183 void *isl_schedule_constraints_free(__isl_take isl_schedule_constraints *sc)
185 enum isl_edge_type i;
187 if (!sc)
188 return NULL;
190 isl_union_set_free(sc->domain);
191 for (i = isl_edge_first; i <= isl_edge_last; ++i)
192 isl_union_map_free(sc->constraint[i]);
194 free(sc);
196 return NULL;
199 isl_ctx *isl_schedule_constraints_get_ctx(
200 __isl_keep isl_schedule_constraints *sc)
202 return sc ? isl_union_set_get_ctx(sc->domain) : NULL;
205 void isl_schedule_constraints_dump(__isl_keep isl_schedule_constraints *sc)
207 if (!sc)
208 return;
210 fprintf(stderr, "domain: ");
211 isl_union_set_dump(sc->domain);
212 fprintf(stderr, "validity: ");
213 isl_union_map_dump(sc->constraint[isl_edge_validity]);
214 fprintf(stderr, "proximity: ");
215 isl_union_map_dump(sc->constraint[isl_edge_proximity]);
216 fprintf(stderr, "coincidence: ");
217 isl_union_map_dump(sc->constraint[isl_edge_coincidence]);
218 fprintf(stderr, "condition: ");
219 isl_union_map_dump(sc->constraint[isl_edge_condition]);
220 fprintf(stderr, "conditional_validity: ");
221 isl_union_map_dump(sc->constraint[isl_edge_conditional_validity]);
224 /* Align the parameters of the fields of "sc".
226 static __isl_give isl_schedule_constraints *
227 isl_schedule_constraints_align_params(__isl_take isl_schedule_constraints *sc)
229 isl_space *space;
230 enum isl_edge_type i;
232 if (!sc)
233 return NULL;
235 space = isl_union_set_get_space(sc->domain);
236 for (i = isl_edge_first; i <= isl_edge_last; ++i)
237 space = isl_space_align_params(space,
238 isl_union_map_get_space(sc->constraint[i]));
240 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
241 sc->constraint[i] = isl_union_map_align_params(
242 sc->constraint[i], isl_space_copy(space));
243 if (!sc->constraint[i])
244 space = isl_space_free(space);
246 sc->domain = isl_union_set_align_params(sc->domain, space);
247 if (!sc->domain)
248 return isl_schedule_constraints_free(sc);
250 return sc;
253 /* Return the total number of isl_maps in the constraints of "sc".
255 static __isl_give int isl_schedule_constraints_n_map(
256 __isl_keep isl_schedule_constraints *sc)
258 enum isl_edge_type i;
259 int n = 0;
261 for (i = isl_edge_first; i <= isl_edge_last; ++i)
262 n += isl_union_map_n_map(sc->constraint[i]);
264 return n;
267 /* Internal information about a node that is used during the construction
268 * of a schedule.
269 * dim represents the space in which the domain lives
270 * sched is a matrix representation of the schedule being constructed
271 * for this node
272 * sched_map is an isl_map representation of the same (partial) schedule
273 * sched_map may be NULL
274 * rank is the number of linearly independent rows in the linear part
275 * of sched
276 * the columns of cmap represent a change of basis for the schedule
277 * coefficients; the first rank columns span the linear part of
278 * the schedule rows
279 * cinv is the inverse of cmap.
280 * start is the first variable in the LP problem in the sequences that
281 * represents the schedule coefficients of this node
282 * nvar is the dimension of the domain
283 * nparam is the number of parameters or 0 if we are not constructing
284 * a parametric schedule
286 * scc is the index of SCC (or WCC) this node belongs to
288 * band contains the band index for each of the rows of the schedule.
289 * band_id is used to differentiate between separate bands at the same
290 * level within the same parent band, i.e., bands that are separated
291 * by the parent band or bands that are independent of each other.
292 * coincident contains a boolean for each of the rows of the schedule,
293 * indicating whether the corresponding scheduling dimension satisfies
294 * the coincidence constraints in the sense that the corresponding
295 * dependence distances are zero.
297 struct isl_sched_node {
298 isl_space *dim;
299 isl_mat *sched;
300 isl_map *sched_map;
301 int rank;
302 isl_mat *cmap;
303 isl_mat *cinv;
304 int start;
305 int nvar;
306 int nparam;
308 int scc;
310 int *band;
311 int *band_id;
312 int *coincident;
315 static int node_has_dim(const void *entry, const void *val)
317 struct isl_sched_node *node = (struct isl_sched_node *)entry;
318 isl_space *dim = (isl_space *)val;
320 return isl_space_is_equal(node->dim, dim);
323 /* An edge in the dependence graph. An edge may be used to
324 * ensure validity of the generated schedule, to minimize the dependence
325 * distance or both
327 * map is the dependence relation, with i -> j in the map if j depends on i
328 * tagged_condition and tagged_validity contain the union of all tagged
329 * condition or conditional validity dependence relations that
330 * specialize the dependence relation "map"; that is,
331 * if (i -> a) -> (j -> b) is an element of "tagged_condition"
332 * or "tagged_validity", then i -> j is an element of "map".
333 * If these fields are NULL, then they represent the empty relation.
334 * src is the source node
335 * dst is the sink node
336 * validity is set if the edge is used to ensure correctness
337 * coincidence is used to enforce zero dependence distances
338 * proximity is set if the edge is used to minimize dependence distances
339 * condition is set if the edge represents a condition
340 * for a conditional validity schedule constraint
341 * local can only be set for condition edges and indicates that
342 * the dependence distance over the edge should be zero
343 * conditional_validity is set if the edge is used to conditionally
344 * ensure correctness
346 * For validity edges, start and end mark the sequence of inequality
347 * constraints in the LP problem that encode the validity constraint
348 * corresponding to this edge.
350 struct isl_sched_edge {
351 isl_map *map;
352 isl_union_map *tagged_condition;
353 isl_union_map *tagged_validity;
355 struct isl_sched_node *src;
356 struct isl_sched_node *dst;
358 unsigned validity : 1;
359 unsigned coincidence : 1;
360 unsigned proximity : 1;
361 unsigned local : 1;
362 unsigned condition : 1;
363 unsigned conditional_validity : 1;
365 int start;
366 int end;
369 /* Internal information about the dependence graph used during
370 * the construction of the schedule.
372 * intra_hmap is a cache, mapping dependence relations to their dual,
373 * for dependences from a node to itself
374 * inter_hmap is a cache, mapping dependence relations to their dual,
375 * for dependences between distinct nodes
377 * n is the number of nodes
378 * node is the list of nodes
379 * maxvar is the maximal number of variables over all nodes
380 * max_row is the allocated number of rows in the schedule
381 * n_row is the current (maximal) number of linearly independent
382 * rows in the node schedules
383 * n_total_row is the current number of rows in the node schedules
384 * n_band is the current number of completed bands
385 * band_start is the starting row in the node schedules of the current band
386 * root is set if this graph is the original dependence graph,
387 * without any splitting
389 * sorted contains a list of node indices sorted according to the
390 * SCC to which a node belongs
392 * n_edge is the number of edges
393 * edge is the list of edges
394 * max_edge contains the maximal number of edges of each type;
395 * in particular, it contains the number of edges in the inital graph.
396 * edge_table contains pointers into the edge array, hashed on the source
397 * and sink spaces; there is one such table for each type;
398 * a given edge may be referenced from more than one table
399 * if the corresponding relation appears in more than of the
400 * sets of dependences
402 * node_table contains pointers into the node array, hashed on the space
404 * region contains a list of variable sequences that should be non-trivial
406 * lp contains the (I)LP problem used to obtain new schedule rows
408 * src_scc and dst_scc are the source and sink SCCs of an edge with
409 * conflicting constraints
411 * scc represents the number of components
413 struct isl_sched_graph {
414 isl_map_to_basic_set *intra_hmap;
415 isl_map_to_basic_set *inter_hmap;
417 struct isl_sched_node *node;
418 int n;
419 int maxvar;
420 int max_row;
421 int n_row;
423 int *sorted;
425 int n_band;
426 int n_total_row;
427 int band_start;
429 int root;
431 struct isl_sched_edge *edge;
432 int n_edge;
433 int max_edge[isl_edge_last + 1];
434 struct isl_hash_table *edge_table[isl_edge_last + 1];
436 struct isl_hash_table *node_table;
437 struct isl_region *region;
439 isl_basic_set *lp;
441 int src_scc;
442 int dst_scc;
444 int scc;
447 /* Initialize node_table based on the list of nodes.
449 static int graph_init_table(isl_ctx *ctx, struct isl_sched_graph *graph)
451 int i;
453 graph->node_table = isl_hash_table_alloc(ctx, graph->n);
454 if (!graph->node_table)
455 return -1;
457 for (i = 0; i < graph->n; ++i) {
458 struct isl_hash_table_entry *entry;
459 uint32_t hash;
461 hash = isl_space_get_hash(graph->node[i].dim);
462 entry = isl_hash_table_find(ctx, graph->node_table, hash,
463 &node_has_dim,
464 graph->node[i].dim, 1);
465 if (!entry)
466 return -1;
467 entry->data = &graph->node[i];
470 return 0;
473 /* Return a pointer to the node that lives within the given space,
474 * or NULL if there is no such node.
476 static struct isl_sched_node *graph_find_node(isl_ctx *ctx,
477 struct isl_sched_graph *graph, __isl_keep isl_space *dim)
479 struct isl_hash_table_entry *entry;
480 uint32_t hash;
482 hash = isl_space_get_hash(dim);
483 entry = isl_hash_table_find(ctx, graph->node_table, hash,
484 &node_has_dim, dim, 0);
486 return entry ? entry->data : NULL;
489 static int edge_has_src_and_dst(const void *entry, const void *val)
491 const struct isl_sched_edge *edge = entry;
492 const struct isl_sched_edge *temp = val;
494 return edge->src == temp->src && edge->dst == temp->dst;
497 /* Add the given edge to graph->edge_table[type].
499 static int graph_edge_table_add(isl_ctx *ctx, struct isl_sched_graph *graph,
500 enum isl_edge_type type, struct isl_sched_edge *edge)
502 struct isl_hash_table_entry *entry;
503 uint32_t hash;
505 hash = isl_hash_init();
506 hash = isl_hash_builtin(hash, edge->src);
507 hash = isl_hash_builtin(hash, edge->dst);
508 entry = isl_hash_table_find(ctx, graph->edge_table[type], hash,
509 &edge_has_src_and_dst, edge, 1);
510 if (!entry)
511 return -1;
512 entry->data = edge;
514 return 0;
517 /* Allocate the edge_tables based on the maximal number of edges of
518 * each type.
520 static int graph_init_edge_tables(isl_ctx *ctx, struct isl_sched_graph *graph)
522 int i;
524 for (i = 0; i <= isl_edge_last; ++i) {
525 graph->edge_table[i] = isl_hash_table_alloc(ctx,
526 graph->max_edge[i]);
527 if (!graph->edge_table[i])
528 return -1;
531 return 0;
534 /* If graph->edge_table[type] contains an edge from the given source
535 * to the given destination, then return the hash table entry of this edge.
536 * Otherwise, return NULL.
538 static struct isl_hash_table_entry *graph_find_edge_entry(
539 struct isl_sched_graph *graph,
540 enum isl_edge_type type,
541 struct isl_sched_node *src, struct isl_sched_node *dst)
543 isl_ctx *ctx = isl_space_get_ctx(src->dim);
544 uint32_t hash;
545 struct isl_sched_edge temp = { .src = src, .dst = dst };
547 hash = isl_hash_init();
548 hash = isl_hash_builtin(hash, temp.src);
549 hash = isl_hash_builtin(hash, temp.dst);
550 return isl_hash_table_find(ctx, graph->edge_table[type], hash,
551 &edge_has_src_and_dst, &temp, 0);
555 /* If graph->edge_table[type] contains an edge from the given source
556 * to the given destination, then return this edge.
557 * Otherwise, return NULL.
559 static struct isl_sched_edge *graph_find_edge(struct isl_sched_graph *graph,
560 enum isl_edge_type type,
561 struct isl_sched_node *src, struct isl_sched_node *dst)
563 struct isl_hash_table_entry *entry;
565 entry = graph_find_edge_entry(graph, type, src, dst);
566 if (!entry)
567 return NULL;
569 return entry->data;
572 /* Check whether the dependence graph has an edge of the given type
573 * between the given two nodes.
575 static int graph_has_edge(struct isl_sched_graph *graph,
576 enum isl_edge_type type,
577 struct isl_sched_node *src, struct isl_sched_node *dst)
579 struct isl_sched_edge *edge;
580 int empty;
582 edge = graph_find_edge(graph, type, src, dst);
583 if (!edge)
584 return 0;
586 empty = isl_map_plain_is_empty(edge->map);
587 if (empty < 0)
588 return -1;
590 return !empty;
593 /* Look for any edge with the same src, dst and map fields as "model".
595 * Return the matching edge if one can be found.
596 * Return "model" if no matching edge is found.
597 * Return NULL on error.
599 static struct isl_sched_edge *graph_find_matching_edge(
600 struct isl_sched_graph *graph, struct isl_sched_edge *model)
602 enum isl_edge_type i;
603 struct isl_sched_edge *edge;
605 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
606 int is_equal;
608 edge = graph_find_edge(graph, i, model->src, model->dst);
609 if (!edge)
610 continue;
611 is_equal = isl_map_plain_is_equal(model->map, edge->map);
612 if (is_equal < 0)
613 return NULL;
614 if (is_equal)
615 return edge;
618 return model;
621 /* Remove the given edge from all the edge_tables that refer to it.
623 static void graph_remove_edge(struct isl_sched_graph *graph,
624 struct isl_sched_edge *edge)
626 isl_ctx *ctx = isl_map_get_ctx(edge->map);
627 enum isl_edge_type i;
629 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
630 struct isl_hash_table_entry *entry;
632 entry = graph_find_edge_entry(graph, i, edge->src, edge->dst);
633 if (!entry)
634 continue;
635 if (entry->data != edge)
636 continue;
637 isl_hash_table_remove(ctx, graph->edge_table[i], entry);
641 /* Check whether the dependence graph has any edge
642 * between the given two nodes.
644 static int graph_has_any_edge(struct isl_sched_graph *graph,
645 struct isl_sched_node *src, struct isl_sched_node *dst)
647 enum isl_edge_type i;
648 int r;
650 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
651 r = graph_has_edge(graph, i, src, dst);
652 if (r < 0 || r)
653 return r;
656 return r;
659 /* Check whether the dependence graph has a validity edge
660 * between the given two nodes.
662 * Conditional validity edges are essentially validity edges that
663 * can be ignored if the corresponding condition edges are iteration private.
664 * Here, we are only checking for the presence of validity
665 * edges, so we need to consider the conditional validity edges too.
666 * In particular, this function is used during the detection
667 * of strongly connected components and we cannot ignore
668 * conditional validity edges during this detection.
670 static int graph_has_validity_edge(struct isl_sched_graph *graph,
671 struct isl_sched_node *src, struct isl_sched_node *dst)
673 int r;
675 r = graph_has_edge(graph, isl_edge_validity, src, dst);
676 if (r < 0 || r)
677 return r;
679 return graph_has_edge(graph, isl_edge_conditional_validity, src, dst);
682 static int graph_alloc(isl_ctx *ctx, struct isl_sched_graph *graph,
683 int n_node, int n_edge)
685 int i;
687 graph->n = n_node;
688 graph->n_edge = n_edge;
689 graph->node = isl_calloc_array(ctx, struct isl_sched_node, graph->n);
690 graph->sorted = isl_calloc_array(ctx, int, graph->n);
691 graph->region = isl_alloc_array(ctx, struct isl_region, graph->n);
692 graph->edge = isl_calloc_array(ctx,
693 struct isl_sched_edge, graph->n_edge);
695 graph->intra_hmap = isl_map_to_basic_set_alloc(ctx, 2 * n_edge);
696 graph->inter_hmap = isl_map_to_basic_set_alloc(ctx, 2 * n_edge);
698 if (!graph->node || !graph->region || (graph->n_edge && !graph->edge) ||
699 !graph->sorted)
700 return -1;
702 for(i = 0; i < graph->n; ++i)
703 graph->sorted[i] = i;
705 return 0;
708 static void graph_free(isl_ctx *ctx, struct isl_sched_graph *graph)
710 int i;
712 isl_map_to_basic_set_free(graph->intra_hmap);
713 isl_map_to_basic_set_free(graph->inter_hmap);
715 for (i = 0; i < graph->n; ++i) {
716 isl_space_free(graph->node[i].dim);
717 isl_mat_free(graph->node[i].sched);
718 isl_map_free(graph->node[i].sched_map);
719 isl_mat_free(graph->node[i].cmap);
720 isl_mat_free(graph->node[i].cinv);
721 if (graph->root) {
722 free(graph->node[i].band);
723 free(graph->node[i].band_id);
724 free(graph->node[i].coincident);
727 free(graph->node);
728 free(graph->sorted);
729 for (i = 0; i < graph->n_edge; ++i) {
730 isl_map_free(graph->edge[i].map);
731 isl_union_map_free(graph->edge[i].tagged_condition);
732 isl_union_map_free(graph->edge[i].tagged_validity);
734 free(graph->edge);
735 free(graph->region);
736 for (i = 0; i <= isl_edge_last; ++i)
737 isl_hash_table_free(ctx, graph->edge_table[i]);
738 isl_hash_table_free(ctx, graph->node_table);
739 isl_basic_set_free(graph->lp);
742 /* For each "set" on which this function is called, increment
743 * graph->n by one and update graph->maxvar.
745 static int init_n_maxvar(__isl_take isl_set *set, void *user)
747 struct isl_sched_graph *graph = user;
748 int nvar = isl_set_dim(set, isl_dim_set);
750 graph->n++;
751 if (nvar > graph->maxvar)
752 graph->maxvar = nvar;
754 isl_set_free(set);
756 return 0;
759 /* Compute the number of rows that should be allocated for the schedule.
760 * The graph can be split at most "n - 1" times, there can be at most
761 * two rows for each dimension in the iteration domains (in particular,
762 * we usually have one row, but it may be split by split_scaled),
763 * and there can be one extra row for ordering the statements.
764 * Note that if we have actually split "n - 1" times, then no ordering
765 * is needed, so in principle we could use "graph->n + 2 * graph->maxvar - 1".
767 static int compute_max_row(struct isl_sched_graph *graph,
768 __isl_keep isl_union_set *domain)
770 graph->n = 0;
771 graph->maxvar = 0;
772 if (isl_union_set_foreach_set(domain, &init_n_maxvar, graph) < 0)
773 return -1;
774 graph->max_row = graph->n + 2 * graph->maxvar;
776 return 0;
779 /* Add a new node to the graph representing the given set.
781 static int extract_node(__isl_take isl_set *set, void *user)
783 int nvar, nparam;
784 isl_ctx *ctx;
785 isl_space *dim;
786 isl_mat *sched;
787 struct isl_sched_graph *graph = user;
788 int *band, *band_id, *coincident;
790 ctx = isl_set_get_ctx(set);
791 dim = isl_set_get_space(set);
792 isl_set_free(set);
793 nvar = isl_space_dim(dim, isl_dim_set);
794 nparam = isl_space_dim(dim, isl_dim_param);
795 if (!ctx->opt->schedule_parametric)
796 nparam = 0;
797 sched = isl_mat_alloc(ctx, 0, 1 + nparam + nvar);
798 graph->node[graph->n].dim = dim;
799 graph->node[graph->n].nvar = nvar;
800 graph->node[graph->n].nparam = nparam;
801 graph->node[graph->n].sched = sched;
802 graph->node[graph->n].sched_map = NULL;
803 band = isl_alloc_array(ctx, int, graph->max_row);
804 graph->node[graph->n].band = band;
805 band_id = isl_calloc_array(ctx, int, graph->max_row);
806 graph->node[graph->n].band_id = band_id;
807 coincident = isl_calloc_array(ctx, int, graph->max_row);
808 graph->node[graph->n].coincident = coincident;
809 graph->n++;
811 if (!sched || (graph->max_row && (!band || !band_id || !coincident)))
812 return -1;
814 return 0;
817 struct isl_extract_edge_data {
818 enum isl_edge_type type;
819 struct isl_sched_graph *graph;
822 /* Merge edge2 into edge1, freeing the contents of edge2.
823 * "type" is the type of the schedule constraint from which edge2 was
824 * extracted.
825 * Return 0 on success and -1 on failure.
827 * edge1 and edge2 are assumed to have the same value for the map field.
829 static int merge_edge(enum isl_edge_type type, struct isl_sched_edge *edge1,
830 struct isl_sched_edge *edge2)
832 edge1->validity |= edge2->validity;
833 edge1->coincidence |= edge2->coincidence;
834 edge1->proximity |= edge2->proximity;
835 edge1->condition |= edge2->condition;
836 edge1->conditional_validity |= edge2->conditional_validity;
837 isl_map_free(edge2->map);
839 if (type == isl_edge_condition) {
840 if (!edge1->tagged_condition)
841 edge1->tagged_condition = edge2->tagged_condition;
842 else
843 edge1->tagged_condition =
844 isl_union_map_union(edge1->tagged_condition,
845 edge2->tagged_condition);
848 if (type == isl_edge_conditional_validity) {
849 if (!edge1->tagged_validity)
850 edge1->tagged_validity = edge2->tagged_validity;
851 else
852 edge1->tagged_validity =
853 isl_union_map_union(edge1->tagged_validity,
854 edge2->tagged_validity);
857 if (type == isl_edge_condition && !edge1->tagged_condition)
858 return -1;
859 if (type == isl_edge_conditional_validity && !edge1->tagged_validity)
860 return -1;
862 return 0;
865 /* Insert dummy tags in domain and range of "map".
867 * In particular, if "map" is of the form
869 * A -> B
871 * then return
873 * [A -> dummy_tag] -> [B -> dummy_tag]
875 * where the dummy_tags are identical and equal to any dummy tags
876 * introduced by any other call to this function.
878 static __isl_give isl_map *insert_dummy_tags(__isl_take isl_map *map)
880 static char dummy;
881 isl_ctx *ctx;
882 isl_id *id;
883 isl_space *space;
884 isl_set *domain, *range;
886 ctx = isl_map_get_ctx(map);
888 id = isl_id_alloc(ctx, NULL, &dummy);
889 space = isl_space_params(isl_map_get_space(map));
890 space = isl_space_set_from_params(space);
891 space = isl_space_set_tuple_id(space, isl_dim_set, id);
892 space = isl_space_map_from_set(space);
894 domain = isl_map_wrap(map);
895 range = isl_map_wrap(isl_map_universe(space));
896 map = isl_map_from_domain_and_range(domain, range);
897 map = isl_map_zip(map);
899 return map;
902 /* Add a new edge to the graph based on the given map
903 * and add it to data->graph->edge_table[data->type].
904 * If a dependence relation of a given type happens to be identical
905 * to one of the dependence relations of a type that was added before,
906 * then we don't create a new edge, but instead mark the original edge
907 * as also representing a dependence of the current type.
909 * Edges of type isl_edge_condition or isl_edge_conditional_validity
910 * may be specified as "tagged" dependence relations. That is, "map"
911 * may contain elements * (i -> a) -> (j -> b), where i -> j denotes
912 * the dependence on iterations and a and b are tags.
913 * edge->map is set to the relation containing the elements i -> j,
914 * while edge->tagged_condition and edge->tagged_validity contain
915 * the union of all the "map" relations
916 * for which extract_edge is called that result in the same edge->map.
918 static int extract_edge(__isl_take isl_map *map, void *user)
920 isl_ctx *ctx = isl_map_get_ctx(map);
921 struct isl_extract_edge_data *data = user;
922 struct isl_sched_graph *graph = data->graph;
923 struct isl_sched_node *src, *dst;
924 isl_space *dim;
925 struct isl_sched_edge *edge;
926 isl_map *tagged = NULL;
928 if (data->type == isl_edge_condition ||
929 data->type == isl_edge_conditional_validity) {
930 if (isl_map_can_zip(map)) {
931 tagged = isl_map_copy(map);
932 map = isl_set_unwrap(isl_map_domain(isl_map_zip(map)));
933 } else {
934 tagged = insert_dummy_tags(isl_map_copy(map));
938 dim = isl_space_domain(isl_map_get_space(map));
939 src = graph_find_node(ctx, graph, dim);
940 isl_space_free(dim);
941 dim = isl_space_range(isl_map_get_space(map));
942 dst = graph_find_node(ctx, graph, dim);
943 isl_space_free(dim);
945 if (!src || !dst) {
946 isl_map_free(map);
947 isl_map_free(tagged);
948 return 0;
951 graph->edge[graph->n_edge].src = src;
952 graph->edge[graph->n_edge].dst = dst;
953 graph->edge[graph->n_edge].map = map;
954 graph->edge[graph->n_edge].validity = 0;
955 graph->edge[graph->n_edge].coincidence = 0;
956 graph->edge[graph->n_edge].proximity = 0;
957 graph->edge[graph->n_edge].condition = 0;
958 graph->edge[graph->n_edge].local = 0;
959 graph->edge[graph->n_edge].conditional_validity = 0;
960 graph->edge[graph->n_edge].tagged_condition = NULL;
961 graph->edge[graph->n_edge].tagged_validity = NULL;
962 if (data->type == isl_edge_validity)
963 graph->edge[graph->n_edge].validity = 1;
964 if (data->type == isl_edge_coincidence)
965 graph->edge[graph->n_edge].coincidence = 1;
966 if (data->type == isl_edge_proximity)
967 graph->edge[graph->n_edge].proximity = 1;
968 if (data->type == isl_edge_condition) {
969 graph->edge[graph->n_edge].condition = 1;
970 graph->edge[graph->n_edge].tagged_condition =
971 isl_union_map_from_map(tagged);
973 if (data->type == isl_edge_conditional_validity) {
974 graph->edge[graph->n_edge].conditional_validity = 1;
975 graph->edge[graph->n_edge].tagged_validity =
976 isl_union_map_from_map(tagged);
979 edge = graph_find_matching_edge(graph, &graph->edge[graph->n_edge]);
980 if (edge == &graph->edge[graph->n_edge])
981 return graph_edge_table_add(ctx, graph, data->type,
982 &graph->edge[graph->n_edge++]);
984 if (merge_edge(data->type, edge, &graph->edge[graph->n_edge]) < 0)
985 return -1;
987 return graph_edge_table_add(ctx, graph, data->type, edge);
990 /* Check whether there is any dependence from node[j] to node[i]
991 * or from node[i] to node[j].
993 static int node_follows_weak(int i, int j, void *user)
995 int f;
996 struct isl_sched_graph *graph = user;
998 f = graph_has_any_edge(graph, &graph->node[j], &graph->node[i]);
999 if (f < 0 || f)
1000 return f;
1001 return graph_has_any_edge(graph, &graph->node[i], &graph->node[j]);
1004 /* Check whether there is a (conditional) validity dependence from node[j]
1005 * to node[i], forcing node[i] to follow node[j].
1007 static int node_follows_strong(int i, int j, void *user)
1009 struct isl_sched_graph *graph = user;
1011 return graph_has_validity_edge(graph, &graph->node[j], &graph->node[i]);
1014 /* Use Tarjan's algorithm for computing the strongly connected components
1015 * in the dependence graph (only validity edges).
1016 * If weak is set, we consider the graph to be undirected and
1017 * we effectively compute the (weakly) connected components.
1018 * Additionally, we also consider other edges when weak is set.
1020 static int detect_ccs(isl_ctx *ctx, struct isl_sched_graph *graph, int weak)
1022 int i, n;
1023 struct isl_tarjan_graph *g = NULL;
1025 g = isl_tarjan_graph_init(ctx, graph->n,
1026 weak ? &node_follows_weak : &node_follows_strong, graph);
1027 if (!g)
1028 return -1;
1030 graph->scc = 0;
1031 i = 0;
1032 n = graph->n;
1033 while (n) {
1034 while (g->order[i] != -1) {
1035 graph->node[g->order[i]].scc = graph->scc;
1036 --n;
1037 ++i;
1039 ++i;
1040 graph->scc++;
1043 isl_tarjan_graph_free(g);
1045 return 0;
1048 /* Apply Tarjan's algorithm to detect the strongly connected components
1049 * in the dependence graph.
1051 static int detect_sccs(isl_ctx *ctx, struct isl_sched_graph *graph)
1053 return detect_ccs(ctx, graph, 0);
1056 /* Apply Tarjan's algorithm to detect the (weakly) connected components
1057 * in the dependence graph.
1059 static int detect_wccs(isl_ctx *ctx, struct isl_sched_graph *graph)
1061 return detect_ccs(ctx, graph, 1);
1064 static int cmp_scc(const void *a, const void *b, void *data)
1066 struct isl_sched_graph *graph = data;
1067 const int *i1 = a;
1068 const int *i2 = b;
1070 return graph->node[*i1].scc - graph->node[*i2].scc;
1073 /* Sort the elements of graph->sorted according to the corresponding SCCs.
1075 static int sort_sccs(struct isl_sched_graph *graph)
1077 return isl_sort(graph->sorted, graph->n, sizeof(int), &cmp_scc, graph);
1080 /* Given a dependence relation R from a node to itself,
1081 * construct the set of coefficients of valid constraints for elements
1082 * in that dependence relation.
1083 * In particular, the result contains tuples of coefficients
1084 * c_0, c_n, c_x such that
1086 * c_0 + c_n n + c_x y - c_x x >= 0 for each (x,y) in R
1088 * or, equivalently,
1090 * c_0 + c_n n + c_x d >= 0 for each d in delta R = { y - x | (x,y) in R }
1092 * We choose here to compute the dual of delta R.
1093 * Alternatively, we could have computed the dual of R, resulting
1094 * in a set of tuples c_0, c_n, c_x, c_y, and then
1095 * plugged in (c_0, c_n, c_x, -c_x).
1097 static __isl_give isl_basic_set *intra_coefficients(
1098 struct isl_sched_graph *graph, __isl_take isl_map *map)
1100 isl_set *delta;
1101 isl_basic_set *coef;
1103 if (isl_map_to_basic_set_has(graph->intra_hmap, map))
1104 return isl_map_to_basic_set_get(graph->intra_hmap, map);
1106 delta = isl_set_remove_divs(isl_map_deltas(isl_map_copy(map)));
1107 coef = isl_set_coefficients(delta);
1108 graph->intra_hmap = isl_map_to_basic_set_set(graph->intra_hmap, map,
1109 isl_basic_set_copy(coef));
1111 return coef;
1114 /* Given a dependence relation R, * construct the set of coefficients
1115 * of valid constraints for elements in that dependence relation.
1116 * In particular, the result contains tuples of coefficients
1117 * c_0, c_n, c_x, c_y such that
1119 * c_0 + c_n n + c_x x + c_y y >= 0 for each (x,y) in R
1122 static __isl_give isl_basic_set *inter_coefficients(
1123 struct isl_sched_graph *graph, __isl_take isl_map *map)
1125 isl_set *set;
1126 isl_basic_set *coef;
1128 if (isl_map_to_basic_set_has(graph->inter_hmap, map))
1129 return isl_map_to_basic_set_get(graph->inter_hmap, map);
1131 set = isl_map_wrap(isl_map_remove_divs(isl_map_copy(map)));
1132 coef = isl_set_coefficients(set);
1133 graph->inter_hmap = isl_map_to_basic_set_set(graph->inter_hmap, map,
1134 isl_basic_set_copy(coef));
1136 return coef;
1139 /* Add constraints to graph->lp that force validity for the given
1140 * dependence from a node i to itself.
1141 * That is, add constraints that enforce
1143 * (c_i_0 + c_i_n n + c_i_x y) - (c_i_0 + c_i_n n + c_i_x x)
1144 * = c_i_x (y - x) >= 0
1146 * for each (x,y) in R.
1147 * We obtain general constraints on coefficients (c_0, c_n, c_x)
1148 * of valid constraints for (y - x) and then plug in (0, 0, c_i_x^+ - c_i_x^-),
1149 * where c_i_x = c_i_x^+ - c_i_x^-, with c_i_x^+ and c_i_x^- non-negative.
1150 * In graph->lp, the c_i_x^- appear before their c_i_x^+ counterpart.
1152 * Actually, we do not construct constraints for the c_i_x themselves,
1153 * but for the coefficients of c_i_x written as a linear combination
1154 * of the columns in node->cmap.
1156 static int add_intra_validity_constraints(struct isl_sched_graph *graph,
1157 struct isl_sched_edge *edge)
1159 unsigned total;
1160 isl_map *map = isl_map_copy(edge->map);
1161 isl_ctx *ctx = isl_map_get_ctx(map);
1162 isl_space *dim;
1163 isl_dim_map *dim_map;
1164 isl_basic_set *coef;
1165 struct isl_sched_node *node = edge->src;
1167 coef = intra_coefficients(graph, map);
1169 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
1171 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1172 isl_space_dim(dim, isl_dim_set), isl_mat_copy(node->cmap));
1173 if (!coef)
1174 goto error;
1176 total = isl_basic_set_total_dim(graph->lp);
1177 dim_map = isl_dim_map_alloc(ctx, total);
1178 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 1, 2,
1179 isl_space_dim(dim, isl_dim_set), 1,
1180 node->nvar, -1);
1181 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 2, 2,
1182 isl_space_dim(dim, isl_dim_set), 1,
1183 node->nvar, 1);
1184 graph->lp = isl_basic_set_extend_constraints(graph->lp,
1185 coef->n_eq, coef->n_ineq);
1186 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
1187 coef, dim_map);
1188 isl_space_free(dim);
1190 return 0;
1191 error:
1192 isl_space_free(dim);
1193 return -1;
1196 /* Add constraints to graph->lp that force validity for the given
1197 * dependence from node i to node j.
1198 * That is, add constraints that enforce
1200 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x) >= 0
1202 * for each (x,y) in R.
1203 * We obtain general constraints on coefficients (c_0, c_n, c_x, c_y)
1204 * of valid constraints for R and then plug in
1205 * (c_j_0 - c_i_0, c_j_n^+ - c_j_n^- - (c_i_n^+ - c_i_n^-),
1206 * c_j_x^+ - c_j_x^- - (c_i_x^+ - c_i_x^-)),
1207 * where c_* = c_*^+ - c_*^-, with c_*^+ and c_*^- non-negative.
1208 * In graph->lp, the c_*^- appear before their c_*^+ counterpart.
1210 * Actually, we do not construct constraints for the c_*_x themselves,
1211 * but for the coefficients of c_*_x written as a linear combination
1212 * of the columns in node->cmap.
1214 static int add_inter_validity_constraints(struct isl_sched_graph *graph,
1215 struct isl_sched_edge *edge)
1217 unsigned total;
1218 isl_map *map = isl_map_copy(edge->map);
1219 isl_ctx *ctx = isl_map_get_ctx(map);
1220 isl_space *dim;
1221 isl_dim_map *dim_map;
1222 isl_basic_set *coef;
1223 struct isl_sched_node *src = edge->src;
1224 struct isl_sched_node *dst = edge->dst;
1226 coef = inter_coefficients(graph, map);
1228 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
1230 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1231 isl_space_dim(dim, isl_dim_set), isl_mat_copy(src->cmap));
1232 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1233 isl_space_dim(dim, isl_dim_set) + src->nvar,
1234 isl_mat_copy(dst->cmap));
1235 if (!coef)
1236 goto error;
1238 total = isl_basic_set_total_dim(graph->lp);
1239 dim_map = isl_dim_map_alloc(ctx, total);
1241 isl_dim_map_range(dim_map, dst->start, 0, 0, 0, 1, 1);
1242 isl_dim_map_range(dim_map, dst->start + 1, 2, 1, 1, dst->nparam, -1);
1243 isl_dim_map_range(dim_map, dst->start + 2, 2, 1, 1, dst->nparam, 1);
1244 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 1, 2,
1245 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
1246 dst->nvar, -1);
1247 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 2, 2,
1248 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
1249 dst->nvar, 1);
1251 isl_dim_map_range(dim_map, src->start, 0, 0, 0, 1, -1);
1252 isl_dim_map_range(dim_map, src->start + 1, 2, 1, 1, src->nparam, 1);
1253 isl_dim_map_range(dim_map, src->start + 2, 2, 1, 1, src->nparam, -1);
1254 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 1, 2,
1255 isl_space_dim(dim, isl_dim_set), 1,
1256 src->nvar, 1);
1257 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 2, 2,
1258 isl_space_dim(dim, isl_dim_set), 1,
1259 src->nvar, -1);
1261 edge->start = graph->lp->n_ineq;
1262 graph->lp = isl_basic_set_extend_constraints(graph->lp,
1263 coef->n_eq, coef->n_ineq);
1264 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
1265 coef, dim_map);
1266 if (!graph->lp)
1267 goto error;
1268 isl_space_free(dim);
1269 edge->end = graph->lp->n_ineq;
1271 return 0;
1272 error:
1273 isl_space_free(dim);
1274 return -1;
1277 /* Add constraints to graph->lp that bound the dependence distance for the given
1278 * dependence from a node i to itself.
1279 * If s = 1, we add the constraint
1281 * c_i_x (y - x) <= m_0 + m_n n
1283 * or
1285 * -c_i_x (y - x) + m_0 + m_n n >= 0
1287 * for each (x,y) in R.
1288 * If s = -1, we add the constraint
1290 * -c_i_x (y - x) <= m_0 + m_n n
1292 * or
1294 * c_i_x (y - x) + m_0 + m_n n >= 0
1296 * for each (x,y) in R.
1297 * We obtain general constraints on coefficients (c_0, c_n, c_x)
1298 * of valid constraints for (y - x) and then plug in (m_0, m_n, -s * c_i_x),
1299 * with each coefficient (except m_0) represented as a pair of non-negative
1300 * coefficients.
1302 * Actually, we do not construct constraints for the c_i_x themselves,
1303 * but for the coefficients of c_i_x written as a linear combination
1304 * of the columns in node->cmap.
1307 * If "local" is set, then we add constraints
1309 * c_i_x (y - x) <= 0
1311 * or
1313 * -c_i_x (y - x) <= 0
1315 * instead, forcing the dependence distance to be (less than or) equal to 0.
1316 * That is, we plug in (0, 0, -s * c_i_x),
1317 * Note that dependences marked local are treated as validity constraints
1318 * by add_all_validity_constraints and therefore also have
1319 * their distances bounded by 0 from below.
1321 static int add_intra_proximity_constraints(struct isl_sched_graph *graph,
1322 struct isl_sched_edge *edge, int s, int local)
1324 unsigned total;
1325 unsigned nparam;
1326 isl_map *map = isl_map_copy(edge->map);
1327 isl_ctx *ctx = isl_map_get_ctx(map);
1328 isl_space *dim;
1329 isl_dim_map *dim_map;
1330 isl_basic_set *coef;
1331 struct isl_sched_node *node = edge->src;
1333 coef = intra_coefficients(graph, map);
1335 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
1337 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1338 isl_space_dim(dim, isl_dim_set), isl_mat_copy(node->cmap));
1339 if (!coef)
1340 goto error;
1342 nparam = isl_space_dim(node->dim, isl_dim_param);
1343 total = isl_basic_set_total_dim(graph->lp);
1344 dim_map = isl_dim_map_alloc(ctx, total);
1346 if (!local) {
1347 isl_dim_map_range(dim_map, 1, 0, 0, 0, 1, 1);
1348 isl_dim_map_range(dim_map, 4, 2, 1, 1, nparam, -1);
1349 isl_dim_map_range(dim_map, 5, 2, 1, 1, nparam, 1);
1351 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 1, 2,
1352 isl_space_dim(dim, isl_dim_set), 1,
1353 node->nvar, s);
1354 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 2, 2,
1355 isl_space_dim(dim, isl_dim_set), 1,
1356 node->nvar, -s);
1357 graph->lp = isl_basic_set_extend_constraints(graph->lp,
1358 coef->n_eq, coef->n_ineq);
1359 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
1360 coef, dim_map);
1361 isl_space_free(dim);
1363 return 0;
1364 error:
1365 isl_space_free(dim);
1366 return -1;
1369 /* Add constraints to graph->lp that bound the dependence distance for the given
1370 * dependence from node i to node j.
1371 * If s = 1, we add the constraint
1373 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x)
1374 * <= m_0 + m_n n
1376 * or
1378 * -(c_j_0 + c_j_n n + c_j_x y) + (c_i_0 + c_i_n n + c_i_x x) +
1379 * m_0 + m_n n >= 0
1381 * for each (x,y) in R.
1382 * If s = -1, we add the constraint
1384 * -((c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x))
1385 * <= m_0 + m_n n
1387 * or
1389 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x) +
1390 * m_0 + m_n n >= 0
1392 * for each (x,y) in R.
1393 * We obtain general constraints on coefficients (c_0, c_n, c_x, c_y)
1394 * of valid constraints for R and then plug in
1395 * (m_0 - s*c_j_0 + s*c_i_0, m_n - s*c_j_n + s*c_i_n,
1396 * -s*c_j_x+s*c_i_x)
1397 * with each coefficient (except m_0, c_j_0 and c_i_0)
1398 * represented as a pair of non-negative coefficients.
1400 * Actually, we do not construct constraints for the c_*_x themselves,
1401 * but for the coefficients of c_*_x written as a linear combination
1402 * of the columns in node->cmap.
1405 * If "local" is set, then we add constraints
1407 * (c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x) <= 0
1409 * or
1411 * -((c_j_0 + c_j_n n + c_j_x y) - (c_i_0 + c_i_n n + c_i_x x)) <= 0
1413 * instead, forcing the dependence distance to be (less than or) equal to 0.
1414 * That is, we plug in
1415 * (-s*c_j_0 + s*c_i_0, -s*c_j_n + s*c_i_n, -s*c_j_x+s*c_i_x).
1416 * Note that dependences marked local are treated as validity constraints
1417 * by add_all_validity_constraints and therefore also have
1418 * their distances bounded by 0 from below.
1420 static int add_inter_proximity_constraints(struct isl_sched_graph *graph,
1421 struct isl_sched_edge *edge, int s, int local)
1423 unsigned total;
1424 unsigned nparam;
1425 isl_map *map = isl_map_copy(edge->map);
1426 isl_ctx *ctx = isl_map_get_ctx(map);
1427 isl_space *dim;
1428 isl_dim_map *dim_map;
1429 isl_basic_set *coef;
1430 struct isl_sched_node *src = edge->src;
1431 struct isl_sched_node *dst = edge->dst;
1433 coef = inter_coefficients(graph, map);
1435 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
1437 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1438 isl_space_dim(dim, isl_dim_set), isl_mat_copy(src->cmap));
1439 coef = isl_basic_set_transform_dims(coef, isl_dim_set,
1440 isl_space_dim(dim, isl_dim_set) + src->nvar,
1441 isl_mat_copy(dst->cmap));
1442 if (!coef)
1443 goto error;
1445 nparam = isl_space_dim(src->dim, isl_dim_param);
1446 total = isl_basic_set_total_dim(graph->lp);
1447 dim_map = isl_dim_map_alloc(ctx, total);
1449 if (!local) {
1450 isl_dim_map_range(dim_map, 1, 0, 0, 0, 1, 1);
1451 isl_dim_map_range(dim_map, 4, 2, 1, 1, nparam, -1);
1452 isl_dim_map_range(dim_map, 5, 2, 1, 1, nparam, 1);
1455 isl_dim_map_range(dim_map, dst->start, 0, 0, 0, 1, -s);
1456 isl_dim_map_range(dim_map, dst->start + 1, 2, 1, 1, dst->nparam, s);
1457 isl_dim_map_range(dim_map, dst->start + 2, 2, 1, 1, dst->nparam, -s);
1458 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 1, 2,
1459 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
1460 dst->nvar, s);
1461 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 2, 2,
1462 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
1463 dst->nvar, -s);
1465 isl_dim_map_range(dim_map, src->start, 0, 0, 0, 1, s);
1466 isl_dim_map_range(dim_map, src->start + 1, 2, 1, 1, src->nparam, -s);
1467 isl_dim_map_range(dim_map, src->start + 2, 2, 1, 1, src->nparam, s);
1468 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 1, 2,
1469 isl_space_dim(dim, isl_dim_set), 1,
1470 src->nvar, -s);
1471 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 2, 2,
1472 isl_space_dim(dim, isl_dim_set), 1,
1473 src->nvar, s);
1475 graph->lp = isl_basic_set_extend_constraints(graph->lp,
1476 coef->n_eq, coef->n_ineq);
1477 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
1478 coef, dim_map);
1479 isl_space_free(dim);
1481 return 0;
1482 error:
1483 isl_space_free(dim);
1484 return -1;
1487 /* Add all validity constraints to graph->lp.
1489 * An edge that is forced to be local needs to have its dependence
1490 * distances equal to zero. We take care of bounding them by 0 from below
1491 * here. add_all_proximity_constraints takes care of bounding them by 0
1492 * from above.
1494 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
1495 * Otherwise, we ignore them.
1497 static int add_all_validity_constraints(struct isl_sched_graph *graph,
1498 int use_coincidence)
1500 int i;
1502 for (i = 0; i < graph->n_edge; ++i) {
1503 struct isl_sched_edge *edge= &graph->edge[i];
1504 int local;
1506 local = edge->local || (edge->coincidence && use_coincidence);
1507 if (!edge->validity && !local)
1508 continue;
1509 if (edge->src != edge->dst)
1510 continue;
1511 if (add_intra_validity_constraints(graph, edge) < 0)
1512 return -1;
1515 for (i = 0; i < graph->n_edge; ++i) {
1516 struct isl_sched_edge *edge = &graph->edge[i];
1517 int local;
1519 local = edge->local || (edge->coincidence && use_coincidence);
1520 if (!edge->validity && !local)
1521 continue;
1522 if (edge->src == edge->dst)
1523 continue;
1524 if (add_inter_validity_constraints(graph, edge) < 0)
1525 return -1;
1528 return 0;
1531 /* Add constraints to graph->lp that bound the dependence distance
1532 * for all dependence relations.
1533 * If a given proximity dependence is identical to a validity
1534 * dependence, then the dependence distance is already bounded
1535 * from below (by zero), so we only need to bound the distance
1536 * from above. (This includes the case of "local" dependences
1537 * which are treated as validity dependence by add_all_validity_constraints.)
1538 * Otherwise, we need to bound the distance both from above and from below.
1540 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
1541 * Otherwise, we ignore them.
1543 static int add_all_proximity_constraints(struct isl_sched_graph *graph,
1544 int use_coincidence)
1546 int i;
1548 for (i = 0; i < graph->n_edge; ++i) {
1549 struct isl_sched_edge *edge= &graph->edge[i];
1550 int local;
1552 local = edge->local || (edge->coincidence && use_coincidence);
1553 if (!edge->proximity && !local)
1554 continue;
1555 if (edge->src == edge->dst &&
1556 add_intra_proximity_constraints(graph, edge, 1, local) < 0)
1557 return -1;
1558 if (edge->src != edge->dst &&
1559 add_inter_proximity_constraints(graph, edge, 1, local) < 0)
1560 return -1;
1561 if (edge->validity || local)
1562 continue;
1563 if (edge->src == edge->dst &&
1564 add_intra_proximity_constraints(graph, edge, -1, 0) < 0)
1565 return -1;
1566 if (edge->src != edge->dst &&
1567 add_inter_proximity_constraints(graph, edge, -1, 0) < 0)
1568 return -1;
1571 return 0;
1574 /* Compute a basis for the rows in the linear part of the schedule
1575 * and extend this basis to a full basis. The remaining rows
1576 * can then be used to force linear independence from the rows
1577 * in the schedule.
1579 * In particular, given the schedule rows S, we compute
1581 * S = H Q
1582 * S U = H
1584 * with H the Hermite normal form of S. That is, all but the
1585 * first rank columns of H are zero and so each row in S is
1586 * a linear combination of the first rank rows of Q.
1587 * The matrix Q is then transposed because we will write the
1588 * coefficients of the next schedule row as a column vector s
1589 * and express this s as a linear combination s = Q c of the
1590 * computed basis.
1591 * Similarly, the matrix U is transposed such that we can
1592 * compute the coefficients c = U s from a schedule row s.
1594 static int node_update_cmap(struct isl_sched_node *node)
1596 isl_mat *H, *U, *Q;
1597 int n_row = isl_mat_rows(node->sched);
1599 H = isl_mat_sub_alloc(node->sched, 0, n_row,
1600 1 + node->nparam, node->nvar);
1602 H = isl_mat_left_hermite(H, 0, &U, &Q);
1603 isl_mat_free(node->cmap);
1604 isl_mat_free(node->cinv);
1605 node->cmap = isl_mat_transpose(Q);
1606 node->cinv = isl_mat_transpose(U);
1607 node->rank = isl_mat_initial_non_zero_cols(H);
1608 isl_mat_free(H);
1610 if (!node->cmap || !node->cinv || node->rank < 0)
1611 return -1;
1612 return 0;
1615 /* How many times should we count the constraints in "edge"?
1617 * If carry is set, then we are counting the number of
1618 * (validity or conditional validity) constraints that will be added
1619 * in setup_carry_lp and we count each edge exactly once.
1621 * Otherwise, we count as follows
1622 * validity -> 1 (>= 0)
1623 * validity+proximity -> 2 (>= 0 and upper bound)
1624 * proximity -> 2 (lower and upper bound)
1625 * local(+any) -> 2 (>= 0 and <= 0)
1627 * If an edge is only marked conditional_validity then it counts
1628 * as zero since it is only checked afterwards.
1630 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
1631 * Otherwise, we ignore them.
1633 static int edge_multiplicity(struct isl_sched_edge *edge, int carry,
1634 int use_coincidence)
1636 if (carry && !edge->validity && !edge->conditional_validity)
1637 return 0;
1638 if (carry)
1639 return 1;
1640 if (edge->proximity || edge->local)
1641 return 2;
1642 if (use_coincidence && edge->coincidence)
1643 return 2;
1644 if (edge->validity)
1645 return 1;
1646 return 0;
1649 /* Count the number of equality and inequality constraints
1650 * that will be added for the given map.
1652 * "use_coincidence" is set if we should take into account coincidence edges.
1654 static int count_map_constraints(struct isl_sched_graph *graph,
1655 struct isl_sched_edge *edge, __isl_take isl_map *map,
1656 int *n_eq, int *n_ineq, int carry, int use_coincidence)
1658 isl_basic_set *coef;
1659 int f = edge_multiplicity(edge, carry, use_coincidence);
1661 if (f == 0) {
1662 isl_map_free(map);
1663 return 0;
1666 if (edge->src == edge->dst)
1667 coef = intra_coefficients(graph, map);
1668 else
1669 coef = inter_coefficients(graph, map);
1670 if (!coef)
1671 return -1;
1672 *n_eq += f * coef->n_eq;
1673 *n_ineq += f * coef->n_ineq;
1674 isl_basic_set_free(coef);
1676 return 0;
1679 /* Count the number of equality and inequality constraints
1680 * that will be added to the main lp problem.
1681 * We count as follows
1682 * validity -> 1 (>= 0)
1683 * validity+proximity -> 2 (>= 0 and upper bound)
1684 * proximity -> 2 (lower and upper bound)
1685 * local(+any) -> 2 (>= 0 and <= 0)
1687 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
1688 * Otherwise, we ignore them.
1690 static int count_constraints(struct isl_sched_graph *graph,
1691 int *n_eq, int *n_ineq, int use_coincidence)
1693 int i;
1695 *n_eq = *n_ineq = 0;
1696 for (i = 0; i < graph->n_edge; ++i) {
1697 struct isl_sched_edge *edge= &graph->edge[i];
1698 isl_map *map = isl_map_copy(edge->map);
1700 if (count_map_constraints(graph, edge, map, n_eq, n_ineq,
1701 0, use_coincidence) < 0)
1702 return -1;
1705 return 0;
1708 /* Count the number of constraints that will be added by
1709 * add_bound_coefficient_constraints and increment *n_eq and *n_ineq
1710 * accordingly.
1712 * In practice, add_bound_coefficient_constraints only adds inequalities.
1714 static int count_bound_coefficient_constraints(isl_ctx *ctx,
1715 struct isl_sched_graph *graph, int *n_eq, int *n_ineq)
1717 int i;
1719 if (ctx->opt->schedule_max_coefficient == -1)
1720 return 0;
1722 for (i = 0; i < graph->n; ++i)
1723 *n_ineq += 2 * graph->node[i].nparam + 2 * graph->node[i].nvar;
1725 return 0;
1728 /* Add constraints that bound the values of the variable and parameter
1729 * coefficients of the schedule.
1731 * The maximal value of the coefficients is defined by the option
1732 * 'schedule_max_coefficient'.
1734 static int add_bound_coefficient_constraints(isl_ctx *ctx,
1735 struct isl_sched_graph *graph)
1737 int i, j, k;
1738 int max_coefficient;
1739 int total;
1741 max_coefficient = ctx->opt->schedule_max_coefficient;
1743 if (max_coefficient == -1)
1744 return 0;
1746 total = isl_basic_set_total_dim(graph->lp);
1748 for (i = 0; i < graph->n; ++i) {
1749 struct isl_sched_node *node = &graph->node[i];
1750 for (j = 0; j < 2 * node->nparam + 2 * node->nvar; ++j) {
1751 int dim;
1752 k = isl_basic_set_alloc_inequality(graph->lp);
1753 if (k < 0)
1754 return -1;
1755 dim = 1 + node->start + 1 + j;
1756 isl_seq_clr(graph->lp->ineq[k], 1 + total);
1757 isl_int_set_si(graph->lp->ineq[k][dim], -1);
1758 isl_int_set_si(graph->lp->ineq[k][0], max_coefficient);
1762 return 0;
1765 /* Construct an ILP problem for finding schedule coefficients
1766 * that result in non-negative, but small dependence distances
1767 * over all dependences.
1768 * In particular, the dependence distances over proximity edges
1769 * are bounded by m_0 + m_n n and we compute schedule coefficients
1770 * with small values (preferably zero) of m_n and m_0.
1772 * All variables of the ILP are non-negative. The actual coefficients
1773 * may be negative, so each coefficient is represented as the difference
1774 * of two non-negative variables. The negative part always appears
1775 * immediately before the positive part.
1776 * Other than that, the variables have the following order
1778 * - sum of positive and negative parts of m_n coefficients
1779 * - m_0
1780 * - sum of positive and negative parts of all c_n coefficients
1781 * (unconstrained when computing non-parametric schedules)
1782 * - sum of positive and negative parts of all c_x coefficients
1783 * - positive and negative parts of m_n coefficients
1784 * - for each node
1785 * - c_i_0
1786 * - positive and negative parts of c_i_n (if parametric)
1787 * - positive and negative parts of c_i_x
1789 * The c_i_x are not represented directly, but through the columns of
1790 * node->cmap. That is, the computed values are for variable t_i_x
1791 * such that c_i_x = Q t_i_x with Q equal to node->cmap.
1793 * The constraints are those from the edges plus two or three equalities
1794 * to express the sums.
1796 * If "use_coincidence" is set, then we treat coincidence edges as local edges.
1797 * Otherwise, we ignore them.
1799 static int setup_lp(isl_ctx *ctx, struct isl_sched_graph *graph,
1800 int use_coincidence)
1802 int i, j;
1803 int k;
1804 unsigned nparam;
1805 unsigned total;
1806 isl_space *dim;
1807 int parametric;
1808 int param_pos;
1809 int n_eq, n_ineq;
1810 int max_constant_term;
1812 max_constant_term = ctx->opt->schedule_max_constant_term;
1814 parametric = ctx->opt->schedule_parametric;
1815 nparam = isl_space_dim(graph->node[0].dim, isl_dim_param);
1816 param_pos = 4;
1817 total = param_pos + 2 * nparam;
1818 for (i = 0; i < graph->n; ++i) {
1819 struct isl_sched_node *node = &graph->node[graph->sorted[i]];
1820 if (node_update_cmap(node) < 0)
1821 return -1;
1822 node->start = total;
1823 total += 1 + 2 * (node->nparam + node->nvar);
1826 if (count_constraints(graph, &n_eq, &n_ineq, use_coincidence) < 0)
1827 return -1;
1828 if (count_bound_coefficient_constraints(ctx, graph, &n_eq, &n_ineq) < 0)
1829 return -1;
1831 dim = isl_space_set_alloc(ctx, 0, total);
1832 isl_basic_set_free(graph->lp);
1833 n_eq += 2 + parametric;
1834 if (max_constant_term != -1)
1835 n_ineq += graph->n;
1837 graph->lp = isl_basic_set_alloc_space(dim, 0, n_eq, n_ineq);
1839 k = isl_basic_set_alloc_equality(graph->lp);
1840 if (k < 0)
1841 return -1;
1842 isl_seq_clr(graph->lp->eq[k], 1 + total);
1843 isl_int_set_si(graph->lp->eq[k][1], -1);
1844 for (i = 0; i < 2 * nparam; ++i)
1845 isl_int_set_si(graph->lp->eq[k][1 + param_pos + i], 1);
1847 if (parametric) {
1848 k = isl_basic_set_alloc_equality(graph->lp);
1849 if (k < 0)
1850 return -1;
1851 isl_seq_clr(graph->lp->eq[k], 1 + total);
1852 isl_int_set_si(graph->lp->eq[k][3], -1);
1853 for (i = 0; i < graph->n; ++i) {
1854 int pos = 1 + graph->node[i].start + 1;
1856 for (j = 0; j < 2 * graph->node[i].nparam; ++j)
1857 isl_int_set_si(graph->lp->eq[k][pos + j], 1);
1861 k = isl_basic_set_alloc_equality(graph->lp);
1862 if (k < 0)
1863 return -1;
1864 isl_seq_clr(graph->lp->eq[k], 1 + total);
1865 isl_int_set_si(graph->lp->eq[k][4], -1);
1866 for (i = 0; i < graph->n; ++i) {
1867 struct isl_sched_node *node = &graph->node[i];
1868 int pos = 1 + node->start + 1 + 2 * node->nparam;
1870 for (j = 0; j < 2 * node->nvar; ++j)
1871 isl_int_set_si(graph->lp->eq[k][pos + j], 1);
1874 if (max_constant_term != -1)
1875 for (i = 0; i < graph->n; ++i) {
1876 struct isl_sched_node *node = &graph->node[i];
1877 k = isl_basic_set_alloc_inequality(graph->lp);
1878 if (k < 0)
1879 return -1;
1880 isl_seq_clr(graph->lp->ineq[k], 1 + total);
1881 isl_int_set_si(graph->lp->ineq[k][1 + node->start], -1);
1882 isl_int_set_si(graph->lp->ineq[k][0], max_constant_term);
1885 if (add_bound_coefficient_constraints(ctx, graph) < 0)
1886 return -1;
1887 if (add_all_validity_constraints(graph, use_coincidence) < 0)
1888 return -1;
1889 if (add_all_proximity_constraints(graph, use_coincidence) < 0)
1890 return -1;
1892 return 0;
1895 /* Analyze the conflicting constraint found by
1896 * isl_tab_basic_set_non_trivial_lexmin. If it corresponds to the validity
1897 * constraint of one of the edges between distinct nodes, living, moreover
1898 * in distinct SCCs, then record the source and sink SCC as this may
1899 * be a good place to cut between SCCs.
1901 static int check_conflict(int con, void *user)
1903 int i;
1904 struct isl_sched_graph *graph = user;
1906 if (graph->src_scc >= 0)
1907 return 0;
1909 con -= graph->lp->n_eq;
1911 if (con >= graph->lp->n_ineq)
1912 return 0;
1914 for (i = 0; i < graph->n_edge; ++i) {
1915 if (!graph->edge[i].validity)
1916 continue;
1917 if (graph->edge[i].src == graph->edge[i].dst)
1918 continue;
1919 if (graph->edge[i].src->scc == graph->edge[i].dst->scc)
1920 continue;
1921 if (graph->edge[i].start > con)
1922 continue;
1923 if (graph->edge[i].end <= con)
1924 continue;
1925 graph->src_scc = graph->edge[i].src->scc;
1926 graph->dst_scc = graph->edge[i].dst->scc;
1929 return 0;
1932 /* Check whether the next schedule row of the given node needs to be
1933 * non-trivial. Lower-dimensional domains may have some trivial rows,
1934 * but as soon as the number of remaining required non-trivial rows
1935 * is as large as the number or remaining rows to be computed,
1936 * all remaining rows need to be non-trivial.
1938 static int needs_row(struct isl_sched_graph *graph, struct isl_sched_node *node)
1940 return node->nvar - node->rank >= graph->maxvar - graph->n_row;
1943 /* Solve the ILP problem constructed in setup_lp.
1944 * For each node such that all the remaining rows of its schedule
1945 * need to be non-trivial, we construct a non-triviality region.
1946 * This region imposes that the next row is independent of previous rows.
1947 * In particular the coefficients c_i_x are represented by t_i_x
1948 * variables with c_i_x = Q t_i_x and Q a unimodular matrix such that
1949 * its first columns span the rows of the previously computed part
1950 * of the schedule. The non-triviality region enforces that at least
1951 * one of the remaining components of t_i_x is non-zero, i.e.,
1952 * that the new schedule row depends on at least one of the remaining
1953 * columns of Q.
1955 static __isl_give isl_vec *solve_lp(struct isl_sched_graph *graph)
1957 int i;
1958 isl_vec *sol;
1959 isl_basic_set *lp;
1961 for (i = 0; i < graph->n; ++i) {
1962 struct isl_sched_node *node = &graph->node[i];
1963 int skip = node->rank;
1964 graph->region[i].pos = node->start + 1 + 2*(node->nparam+skip);
1965 if (needs_row(graph, node))
1966 graph->region[i].len = 2 * (node->nvar - skip);
1967 else
1968 graph->region[i].len = 0;
1970 lp = isl_basic_set_copy(graph->lp);
1971 sol = isl_tab_basic_set_non_trivial_lexmin(lp, 2, graph->n,
1972 graph->region, &check_conflict, graph);
1973 return sol;
1976 /* Update the schedules of all nodes based on the given solution
1977 * of the LP problem.
1978 * The new row is added to the current band.
1979 * All possibly negative coefficients are encoded as a difference
1980 * of two non-negative variables, so we need to perform the subtraction
1981 * here. Moreover, if use_cmap is set, then the solution does
1982 * not refer to the actual coefficients c_i_x, but instead to variables
1983 * t_i_x such that c_i_x = Q t_i_x and Q is equal to node->cmap.
1984 * In this case, we then also need to perform this multiplication
1985 * to obtain the values of c_i_x.
1987 * If coincident is set, then the caller guarantees that the new
1988 * row satisfies the coincidence constraints.
1990 static int update_schedule(struct isl_sched_graph *graph,
1991 __isl_take isl_vec *sol, int use_cmap, int coincident)
1993 int i, j;
1994 isl_vec *csol = NULL;
1996 if (!sol)
1997 goto error;
1998 if (sol->size == 0)
1999 isl_die(sol->ctx, isl_error_internal,
2000 "no solution found", goto error);
2001 if (graph->n_total_row >= graph->max_row)
2002 isl_die(sol->ctx, isl_error_internal,
2003 "too many schedule rows", goto error);
2005 for (i = 0; i < graph->n; ++i) {
2006 struct isl_sched_node *node = &graph->node[i];
2007 int pos = node->start;
2008 int row = isl_mat_rows(node->sched);
2010 isl_vec_free(csol);
2011 csol = isl_vec_alloc(sol->ctx, node->nvar);
2012 if (!csol)
2013 goto error;
2015 isl_map_free(node->sched_map);
2016 node->sched_map = NULL;
2017 node->sched = isl_mat_add_rows(node->sched, 1);
2018 if (!node->sched)
2019 goto error;
2020 node->sched = isl_mat_set_element(node->sched, row, 0,
2021 sol->el[1 + pos]);
2022 for (j = 0; j < node->nparam + node->nvar; ++j)
2023 isl_int_sub(sol->el[1 + pos + 1 + 2 * j + 1],
2024 sol->el[1 + pos + 1 + 2 * j + 1],
2025 sol->el[1 + pos + 1 + 2 * j]);
2026 for (j = 0; j < node->nparam; ++j)
2027 node->sched = isl_mat_set_element(node->sched,
2028 row, 1 + j, sol->el[1+pos+1+2*j+1]);
2029 for (j = 0; j < node->nvar; ++j)
2030 isl_int_set(csol->el[j],
2031 sol->el[1+pos+1+2*(node->nparam+j)+1]);
2032 if (use_cmap)
2033 csol = isl_mat_vec_product(isl_mat_copy(node->cmap),
2034 csol);
2035 if (!csol)
2036 goto error;
2037 for (j = 0; j < node->nvar; ++j)
2038 node->sched = isl_mat_set_element(node->sched,
2039 row, 1 + node->nparam + j, csol->el[j]);
2040 node->band[graph->n_total_row] = graph->n_band;
2041 node->coincident[graph->n_total_row] = coincident;
2043 isl_vec_free(sol);
2044 isl_vec_free(csol);
2046 graph->n_row++;
2047 graph->n_total_row++;
2049 return 0;
2050 error:
2051 isl_vec_free(sol);
2052 isl_vec_free(csol);
2053 return -1;
2056 /* Convert row "row" of node->sched into an isl_aff living in "ls"
2057 * and return this isl_aff.
2059 static __isl_give isl_aff *extract_schedule_row(__isl_take isl_local_space *ls,
2060 struct isl_sched_node *node, int row)
2062 int j;
2063 isl_int v;
2064 isl_aff *aff;
2066 isl_int_init(v);
2068 aff = isl_aff_zero_on_domain(ls);
2069 isl_mat_get_element(node->sched, row, 0, &v);
2070 aff = isl_aff_set_constant(aff, v);
2071 for (j = 0; j < node->nparam; ++j) {
2072 isl_mat_get_element(node->sched, row, 1 + j, &v);
2073 aff = isl_aff_set_coefficient(aff, isl_dim_param, j, v);
2075 for (j = 0; j < node->nvar; ++j) {
2076 isl_mat_get_element(node->sched, row, 1 + node->nparam + j, &v);
2077 aff = isl_aff_set_coefficient(aff, isl_dim_in, j, v);
2080 isl_int_clear(v);
2082 return aff;
2085 /* Convert node->sched into a multi_aff and return this multi_aff.
2087 static __isl_give isl_multi_aff *node_extract_schedule_multi_aff(
2088 struct isl_sched_node *node)
2090 int i;
2091 isl_space *space;
2092 isl_local_space *ls;
2093 isl_aff *aff;
2094 isl_multi_aff *ma;
2095 int nrow, ncol;
2097 nrow = isl_mat_rows(node->sched);
2098 ncol = isl_mat_cols(node->sched) - 1;
2099 space = isl_space_from_domain(isl_space_copy(node->dim));
2100 space = isl_space_add_dims(space, isl_dim_out, nrow);
2101 ma = isl_multi_aff_zero(space);
2102 ls = isl_local_space_from_space(isl_space_copy(node->dim));
2104 for (i = 0; i < nrow; ++i) {
2105 aff = extract_schedule_row(isl_local_space_copy(ls), node, i);
2106 ma = isl_multi_aff_set_aff(ma, i, aff);
2109 isl_local_space_free(ls);
2111 return ma;
2114 /* Convert node->sched into a map and return this map.
2116 * The result is cached in node->sched_map, which needs to be released
2117 * whenever node->sched is updated.
2119 static __isl_give isl_map *node_extract_schedule(struct isl_sched_node *node)
2121 if (!node->sched_map) {
2122 isl_multi_aff *ma;
2124 ma = node_extract_schedule_multi_aff(node);
2125 node->sched_map = isl_map_from_multi_aff(ma);
2128 return isl_map_copy(node->sched_map);
2131 /* Construct a map that can be used to update dependence relation
2132 * based on the current schedule.
2133 * That is, construct a map expressing that source and sink
2134 * are executed within the same iteration of the current schedule.
2135 * This map can then be intersected with the dependence relation.
2136 * This is not the most efficient way, but this shouldn't be a critical
2137 * operation.
2139 static __isl_give isl_map *specializer(struct isl_sched_node *src,
2140 struct isl_sched_node *dst)
2142 isl_map *src_sched, *dst_sched;
2144 src_sched = node_extract_schedule(src);
2145 dst_sched = node_extract_schedule(dst);
2146 return isl_map_apply_range(src_sched, isl_map_reverse(dst_sched));
2149 /* Intersect the domains of the nested relations in domain and range
2150 * of "umap" with "map".
2152 static __isl_give isl_union_map *intersect_domains(
2153 __isl_take isl_union_map *umap, __isl_keep isl_map *map)
2155 isl_union_set *uset;
2157 umap = isl_union_map_zip(umap);
2158 uset = isl_union_set_from_set(isl_map_wrap(isl_map_copy(map)));
2159 umap = isl_union_map_intersect_domain(umap, uset);
2160 umap = isl_union_map_zip(umap);
2161 return umap;
2164 /* Update the dependence relation of the given edge based
2165 * on the current schedule.
2166 * If the dependence is carried completely by the current schedule, then
2167 * it is removed from the edge_tables. It is kept in the list of edges
2168 * as otherwise all edge_tables would have to be recomputed.
2170 static int update_edge(struct isl_sched_graph *graph,
2171 struct isl_sched_edge *edge)
2173 isl_map *id;
2175 id = specializer(edge->src, edge->dst);
2176 edge->map = isl_map_intersect(edge->map, isl_map_copy(id));
2177 if (!edge->map)
2178 goto error;
2180 if (edge->tagged_condition) {
2181 edge->tagged_condition =
2182 intersect_domains(edge->tagged_condition, id);
2183 if (!edge->tagged_condition)
2184 goto error;
2186 if (edge->tagged_validity) {
2187 edge->tagged_validity =
2188 intersect_domains(edge->tagged_validity, id);
2189 if (!edge->tagged_validity)
2190 goto error;
2193 isl_map_free(id);
2194 if (isl_map_plain_is_empty(edge->map))
2195 graph_remove_edge(graph, edge);
2197 return 0;
2198 error:
2199 isl_map_free(id);
2200 return -1;
2203 /* Update the dependence relations of all edges based on the current schedule.
2205 static int update_edges(isl_ctx *ctx, struct isl_sched_graph *graph)
2207 int i;
2209 for (i = graph->n_edge - 1; i >= 0; --i) {
2210 if (update_edge(graph, &graph->edge[i]) < 0)
2211 return -1;
2214 return 0;
2217 static void next_band(struct isl_sched_graph *graph)
2219 graph->band_start = graph->n_total_row;
2220 graph->n_band++;
2223 /* Topologically sort statements mapped to the same schedule iteration
2224 * and add a row to the schedule corresponding to this order.
2226 static int sort_statements(isl_ctx *ctx, struct isl_sched_graph *graph)
2228 int i, j;
2230 if (graph->n <= 1)
2231 return 0;
2233 if (update_edges(ctx, graph) < 0)
2234 return -1;
2236 if (graph->n_edge == 0)
2237 return 0;
2239 if (detect_sccs(ctx, graph) < 0)
2240 return -1;
2242 if (graph->n_total_row >= graph->max_row)
2243 isl_die(ctx, isl_error_internal,
2244 "too many schedule rows", return -1);
2246 for (i = 0; i < graph->n; ++i) {
2247 struct isl_sched_node *node = &graph->node[i];
2248 int row = isl_mat_rows(node->sched);
2249 int cols = isl_mat_cols(node->sched);
2251 isl_map_free(node->sched_map);
2252 node->sched_map = NULL;
2253 node->sched = isl_mat_add_rows(node->sched, 1);
2254 if (!node->sched)
2255 return -1;
2256 node->sched = isl_mat_set_element_si(node->sched, row, 0,
2257 node->scc);
2258 for (j = 1; j < cols; ++j)
2259 node->sched = isl_mat_set_element_si(node->sched,
2260 row, j, 0);
2261 node->band[graph->n_total_row] = graph->n_band;
2264 graph->n_total_row++;
2265 next_band(graph);
2267 return 0;
2270 /* Construct an isl_schedule based on the computed schedule stored
2271 * in graph and with parameters specified by dim.
2273 static __isl_give isl_schedule *extract_schedule(struct isl_sched_graph *graph,
2274 __isl_take isl_space *dim)
2276 int i;
2277 isl_ctx *ctx;
2278 isl_schedule *sched = NULL;
2280 if (!dim)
2281 return NULL;
2283 ctx = isl_space_get_ctx(dim);
2284 sched = isl_calloc(ctx, struct isl_schedule,
2285 sizeof(struct isl_schedule) +
2286 (graph->n - 1) * sizeof(struct isl_schedule_node));
2287 if (!sched)
2288 goto error;
2290 sched->ref = 1;
2291 sched->n = graph->n;
2292 sched->n_band = graph->n_band;
2293 sched->n_total_row = graph->n_total_row;
2295 for (i = 0; i < sched->n; ++i) {
2296 int r, b;
2297 int *band_end, *band_id, *coincident;
2299 sched->node[i].sched =
2300 node_extract_schedule_multi_aff(&graph->node[i]);
2301 if (!sched->node[i].sched)
2302 goto error;
2304 sched->node[i].n_band = graph->n_band;
2305 if (graph->n_band == 0)
2306 continue;
2308 band_end = isl_alloc_array(ctx, int, graph->n_band);
2309 band_id = isl_alloc_array(ctx, int, graph->n_band);
2310 coincident = isl_alloc_array(ctx, int, graph->n_total_row);
2311 sched->node[i].band_end = band_end;
2312 sched->node[i].band_id = band_id;
2313 sched->node[i].coincident = coincident;
2314 if (!band_end || !band_id || !coincident)
2315 goto error;
2317 for (r = 0; r < graph->n_total_row; ++r)
2318 coincident[r] = graph->node[i].coincident[r];
2319 for (r = b = 0; r < graph->n_total_row; ++r) {
2320 if (graph->node[i].band[r] == b)
2321 continue;
2322 band_end[b++] = r;
2323 if (graph->node[i].band[r] == -1)
2324 break;
2326 if (r == graph->n_total_row)
2327 band_end[b++] = r;
2328 sched->node[i].n_band = b;
2329 for (--b; b >= 0; --b)
2330 band_id[b] = graph->node[i].band_id[b];
2333 sched->dim = dim;
2335 return sched;
2336 error:
2337 isl_space_free(dim);
2338 isl_schedule_free(sched);
2339 return NULL;
2342 /* Copy nodes that satisfy node_pred from the src dependence graph
2343 * to the dst dependence graph.
2345 static int copy_nodes(struct isl_sched_graph *dst, struct isl_sched_graph *src,
2346 int (*node_pred)(struct isl_sched_node *node, int data), int data)
2348 int i;
2350 dst->n = 0;
2351 for (i = 0; i < src->n; ++i) {
2352 if (!node_pred(&src->node[i], data))
2353 continue;
2354 dst->node[dst->n].dim = isl_space_copy(src->node[i].dim);
2355 dst->node[dst->n].nvar = src->node[i].nvar;
2356 dst->node[dst->n].nparam = src->node[i].nparam;
2357 dst->node[dst->n].sched = isl_mat_copy(src->node[i].sched);
2358 dst->node[dst->n].sched_map =
2359 isl_map_copy(src->node[i].sched_map);
2360 dst->node[dst->n].band = src->node[i].band;
2361 dst->node[dst->n].band_id = src->node[i].band_id;
2362 dst->node[dst->n].coincident = src->node[i].coincident;
2363 dst->n++;
2366 return 0;
2369 /* Copy non-empty edges that satisfy edge_pred from the src dependence graph
2370 * to the dst dependence graph.
2371 * If the source or destination node of the edge is not in the destination
2372 * graph, then it must be a backward proximity edge and it should simply
2373 * be ignored.
2375 static int copy_edges(isl_ctx *ctx, struct isl_sched_graph *dst,
2376 struct isl_sched_graph *src,
2377 int (*edge_pred)(struct isl_sched_edge *edge, int data), int data)
2379 int i;
2380 enum isl_edge_type t;
2382 dst->n_edge = 0;
2383 for (i = 0; i < src->n_edge; ++i) {
2384 struct isl_sched_edge *edge = &src->edge[i];
2385 isl_map *map;
2386 isl_union_map *tagged_condition;
2387 isl_union_map *tagged_validity;
2388 struct isl_sched_node *dst_src, *dst_dst;
2390 if (!edge_pred(edge, data))
2391 continue;
2393 if (isl_map_plain_is_empty(edge->map))
2394 continue;
2396 dst_src = graph_find_node(ctx, dst, edge->src->dim);
2397 dst_dst = graph_find_node(ctx, dst, edge->dst->dim);
2398 if (!dst_src || !dst_dst) {
2399 if (edge->validity || edge->conditional_validity)
2400 isl_die(ctx, isl_error_internal,
2401 "backward (conditional) validity edge",
2402 return -1);
2403 continue;
2406 map = isl_map_copy(edge->map);
2407 tagged_condition = isl_union_map_copy(edge->tagged_condition);
2408 tagged_validity = isl_union_map_copy(edge->tagged_validity);
2410 dst->edge[dst->n_edge].src = dst_src;
2411 dst->edge[dst->n_edge].dst = dst_dst;
2412 dst->edge[dst->n_edge].map = map;
2413 dst->edge[dst->n_edge].tagged_condition = tagged_condition;
2414 dst->edge[dst->n_edge].tagged_validity = tagged_validity;
2415 dst->edge[dst->n_edge].validity = edge->validity;
2416 dst->edge[dst->n_edge].proximity = edge->proximity;
2417 dst->edge[dst->n_edge].coincidence = edge->coincidence;
2418 dst->edge[dst->n_edge].condition = edge->condition;
2419 dst->edge[dst->n_edge].conditional_validity =
2420 edge->conditional_validity;
2421 dst->n_edge++;
2423 if (edge->tagged_condition && !tagged_condition)
2424 return -1;
2425 if (edge->tagged_validity && !tagged_validity)
2426 return -1;
2428 for (t = isl_edge_first; t <= isl_edge_last; ++t) {
2429 if (edge !=
2430 graph_find_edge(src, t, edge->src, edge->dst))
2431 continue;
2432 if (graph_edge_table_add(ctx, dst, t,
2433 &dst->edge[dst->n_edge - 1]) < 0)
2434 return -1;
2438 return 0;
2441 /* Given a "src" dependence graph that contains the nodes from "dst"
2442 * that satisfy node_pred, copy the schedule computed in "src"
2443 * for those nodes back to "dst".
2445 static int copy_schedule(struct isl_sched_graph *dst,
2446 struct isl_sched_graph *src,
2447 int (*node_pred)(struct isl_sched_node *node, int data), int data)
2449 int i;
2451 src->n = 0;
2452 for (i = 0; i < dst->n; ++i) {
2453 if (!node_pred(&dst->node[i], data))
2454 continue;
2455 isl_mat_free(dst->node[i].sched);
2456 isl_map_free(dst->node[i].sched_map);
2457 dst->node[i].sched = isl_mat_copy(src->node[src->n].sched);
2458 dst->node[i].sched_map =
2459 isl_map_copy(src->node[src->n].sched_map);
2460 src->n++;
2463 dst->max_row = src->max_row;
2464 dst->n_total_row = src->n_total_row;
2465 dst->n_band = src->n_band;
2467 return 0;
2470 /* Compute the maximal number of variables over all nodes.
2471 * This is the maximal number of linearly independent schedule
2472 * rows that we need to compute.
2473 * Just in case we end up in a part of the dependence graph
2474 * with only lower-dimensional domains, we make sure we will
2475 * compute the required amount of extra linearly independent rows.
2477 static int compute_maxvar(struct isl_sched_graph *graph)
2479 int i;
2481 graph->maxvar = 0;
2482 for (i = 0; i < graph->n; ++i) {
2483 struct isl_sched_node *node = &graph->node[i];
2484 int nvar;
2486 if (node_update_cmap(node) < 0)
2487 return -1;
2488 nvar = node->nvar + graph->n_row - node->rank;
2489 if (nvar > graph->maxvar)
2490 graph->maxvar = nvar;
2493 return 0;
2496 static int compute_schedule(isl_ctx *ctx, struct isl_sched_graph *graph);
2497 static int compute_schedule_wcc(isl_ctx *ctx, struct isl_sched_graph *graph);
2499 /* Compute a schedule for a subgraph of "graph". In particular, for
2500 * the graph composed of nodes that satisfy node_pred and edges that
2501 * that satisfy edge_pred. The caller should precompute the number
2502 * of nodes and edges that satisfy these predicates and pass them along
2503 * as "n" and "n_edge".
2504 * If the subgraph is known to consist of a single component, then wcc should
2505 * be set and then we call compute_schedule_wcc on the constructed subgraph.
2506 * Otherwise, we call compute_schedule, which will check whether the subgraph
2507 * is connected.
2509 static int compute_sub_schedule(isl_ctx *ctx,
2510 struct isl_sched_graph *graph, int n, int n_edge,
2511 int (*node_pred)(struct isl_sched_node *node, int data),
2512 int (*edge_pred)(struct isl_sched_edge *edge, int data),
2513 int data, int wcc)
2515 struct isl_sched_graph split = { 0 };
2516 int t;
2518 if (graph_alloc(ctx, &split, n, n_edge) < 0)
2519 goto error;
2520 if (copy_nodes(&split, graph, node_pred, data) < 0)
2521 goto error;
2522 if (graph_init_table(ctx, &split) < 0)
2523 goto error;
2524 for (t = 0; t <= isl_edge_last; ++t)
2525 split.max_edge[t] = graph->max_edge[t];
2526 if (graph_init_edge_tables(ctx, &split) < 0)
2527 goto error;
2528 if (copy_edges(ctx, &split, graph, edge_pred, data) < 0)
2529 goto error;
2530 split.n_row = graph->n_row;
2531 split.max_row = graph->max_row;
2532 split.n_total_row = graph->n_total_row;
2533 split.n_band = graph->n_band;
2534 split.band_start = graph->band_start;
2536 if (wcc && compute_schedule_wcc(ctx, &split) < 0)
2537 goto error;
2538 if (!wcc && compute_schedule(ctx, &split) < 0)
2539 goto error;
2541 copy_schedule(graph, &split, node_pred, data);
2543 graph_free(ctx, &split);
2544 return 0;
2545 error:
2546 graph_free(ctx, &split);
2547 return -1;
2550 static int node_scc_exactly(struct isl_sched_node *node, int scc)
2552 return node->scc == scc;
2555 static int node_scc_at_most(struct isl_sched_node *node, int scc)
2557 return node->scc <= scc;
2560 static int node_scc_at_least(struct isl_sched_node *node, int scc)
2562 return node->scc >= scc;
2565 static int edge_scc_exactly(struct isl_sched_edge *edge, int scc)
2567 return edge->src->scc == scc && edge->dst->scc == scc;
2570 static int edge_dst_scc_at_most(struct isl_sched_edge *edge, int scc)
2572 return edge->dst->scc <= scc;
2575 static int edge_src_scc_at_least(struct isl_sched_edge *edge, int scc)
2577 return edge->src->scc >= scc;
2580 /* Pad the schedules of all nodes with zero rows such that in the end
2581 * they all have graph->n_total_row rows.
2582 * The extra rows don't belong to any band, so they get assigned band number -1.
2584 static int pad_schedule(struct isl_sched_graph *graph)
2586 int i, j;
2588 for (i = 0; i < graph->n; ++i) {
2589 struct isl_sched_node *node = &graph->node[i];
2590 int row = isl_mat_rows(node->sched);
2591 if (graph->n_total_row > row) {
2592 isl_map_free(node->sched_map);
2593 node->sched_map = NULL;
2595 node->sched = isl_mat_add_zero_rows(node->sched,
2596 graph->n_total_row - row);
2597 if (!node->sched)
2598 return -1;
2599 for (j = row; j < graph->n_total_row; ++j)
2600 node->band[j] = -1;
2603 return 0;
2606 /* Reset the current band by dropping all its schedule rows.
2608 static int reset_band(struct isl_sched_graph *graph)
2610 int i;
2611 int drop;
2613 drop = graph->n_total_row - graph->band_start;
2614 graph->n_total_row -= drop;
2615 graph->n_row -= drop;
2617 for (i = 0; i < graph->n; ++i) {
2618 struct isl_sched_node *node = &graph->node[i];
2620 isl_map_free(node->sched_map);
2621 node->sched_map = NULL;
2623 node->sched = isl_mat_drop_rows(node->sched,
2624 graph->band_start, drop);
2626 if (!node->sched)
2627 return -1;
2630 return 0;
2633 /* Split the current graph into two parts and compute a schedule for each
2634 * part individually. In particular, one part consists of all SCCs up
2635 * to and including graph->src_scc, while the other part contains the other
2636 * SCCS.
2638 * The split is enforced in the schedule by constant rows with two different
2639 * values (0 and 1). These constant rows replace the previously computed rows
2640 * in the current band.
2641 * It would be possible to reuse them as the first rows in the next
2642 * band, but recomputing them may result in better rows as we are looking
2643 * at a smaller part of the dependence graph.
2645 * Since we do not enforce coincidence, we conservatively mark the
2646 * splitting row as not coincident.
2648 * The band_id of the second group is set to n, where n is the number
2649 * of nodes in the first group. This ensures that the band_ids over
2650 * the two groups remain disjoint, even if either or both of the two
2651 * groups contain independent components.
2653 static int compute_split_schedule(isl_ctx *ctx, struct isl_sched_graph *graph)
2655 int i, j, n, e1, e2;
2656 int n_total_row, orig_total_row;
2657 int n_band, orig_band;
2659 if (graph->n_total_row >= graph->max_row)
2660 isl_die(ctx, isl_error_internal,
2661 "too many schedule rows", return -1);
2663 if (reset_band(graph) < 0)
2664 return -1;
2666 n = 0;
2667 for (i = 0; i < graph->n; ++i) {
2668 struct isl_sched_node *node = &graph->node[i];
2669 int row = isl_mat_rows(node->sched);
2670 int cols = isl_mat_cols(node->sched);
2671 int before = node->scc <= graph->src_scc;
2673 if (before)
2674 n++;
2676 isl_map_free(node->sched_map);
2677 node->sched_map = NULL;
2678 node->sched = isl_mat_add_rows(node->sched, 1);
2679 if (!node->sched)
2680 return -1;
2681 node->sched = isl_mat_set_element_si(node->sched, row, 0,
2682 !before);
2683 for (j = 1; j < cols; ++j)
2684 node->sched = isl_mat_set_element_si(node->sched,
2685 row, j, 0);
2686 node->band[graph->n_total_row] = graph->n_band;
2687 node->coincident[graph->n_total_row] = 0;
2690 e1 = e2 = 0;
2691 for (i = 0; i < graph->n_edge; ++i) {
2692 if (graph->edge[i].dst->scc <= graph->src_scc)
2693 e1++;
2694 if (graph->edge[i].src->scc > graph->src_scc)
2695 e2++;
2698 graph->n_total_row++;
2699 next_band(graph);
2701 for (i = 0; i < graph->n; ++i) {
2702 struct isl_sched_node *node = &graph->node[i];
2703 if (node->scc > graph->src_scc)
2704 node->band_id[graph->n_band] = n;
2707 orig_total_row = graph->n_total_row;
2708 orig_band = graph->n_band;
2709 if (compute_sub_schedule(ctx, graph, n, e1,
2710 &node_scc_at_most, &edge_dst_scc_at_most,
2711 graph->src_scc, 0) < 0)
2712 return -1;
2713 n_total_row = graph->n_total_row;
2714 graph->n_total_row = orig_total_row;
2715 n_band = graph->n_band;
2716 graph->n_band = orig_band;
2717 if (compute_sub_schedule(ctx, graph, graph->n - n, e2,
2718 &node_scc_at_least, &edge_src_scc_at_least,
2719 graph->src_scc + 1, 0) < 0)
2720 return -1;
2721 if (n_total_row > graph->n_total_row)
2722 graph->n_total_row = n_total_row;
2723 if (n_band > graph->n_band)
2724 graph->n_band = n_band;
2726 return pad_schedule(graph);
2729 /* Compute the next band of the schedule after updating the dependence
2730 * relations based on the the current schedule.
2732 static int compute_next_band(isl_ctx *ctx, struct isl_sched_graph *graph)
2734 if (update_edges(ctx, graph) < 0)
2735 return -1;
2736 next_band(graph);
2738 return compute_schedule(ctx, graph);
2741 /* Add constraints to graph->lp that force the dependence "map" (which
2742 * is part of the dependence relation of "edge")
2743 * to be respected and attempt to carry it, where the edge is one from
2744 * a node j to itself. "pos" is the sequence number of the given map.
2745 * That is, add constraints that enforce
2747 * (c_j_0 + c_j_n n + c_j_x y) - (c_j_0 + c_j_n n + c_j_x x)
2748 * = c_j_x (y - x) >= e_i
2750 * for each (x,y) in R.
2751 * We obtain general constraints on coefficients (c_0, c_n, c_x)
2752 * of valid constraints for (y - x) and then plug in (-e_i, 0, c_j_x),
2753 * with each coefficient in c_j_x represented as a pair of non-negative
2754 * coefficients.
2756 static int add_intra_constraints(struct isl_sched_graph *graph,
2757 struct isl_sched_edge *edge, __isl_take isl_map *map, int pos)
2759 unsigned total;
2760 isl_ctx *ctx = isl_map_get_ctx(map);
2761 isl_space *dim;
2762 isl_dim_map *dim_map;
2763 isl_basic_set *coef;
2764 struct isl_sched_node *node = edge->src;
2766 coef = intra_coefficients(graph, map);
2767 if (!coef)
2768 return -1;
2770 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
2772 total = isl_basic_set_total_dim(graph->lp);
2773 dim_map = isl_dim_map_alloc(ctx, total);
2774 isl_dim_map_range(dim_map, 3 + pos, 0, 0, 0, 1, -1);
2775 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 1, 2,
2776 isl_space_dim(dim, isl_dim_set), 1,
2777 node->nvar, -1);
2778 isl_dim_map_range(dim_map, node->start + 2 * node->nparam + 2, 2,
2779 isl_space_dim(dim, isl_dim_set), 1,
2780 node->nvar, 1);
2781 graph->lp = isl_basic_set_extend_constraints(graph->lp,
2782 coef->n_eq, coef->n_ineq);
2783 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
2784 coef, dim_map);
2785 isl_space_free(dim);
2787 return 0;
2790 /* Add constraints to graph->lp that force the dependence "map" (which
2791 * is part of the dependence relation of "edge")
2792 * to be respected and attempt to carry it, where the edge is one from
2793 * node j to node k. "pos" is the sequence number of the given map.
2794 * That is, add constraints that enforce
2796 * (c_k_0 + c_k_n n + c_k_x y) - (c_j_0 + c_j_n n + c_j_x x) >= e_i
2798 * for each (x,y) in R.
2799 * We obtain general constraints on coefficients (c_0, c_n, c_x)
2800 * of valid constraints for R and then plug in
2801 * (-e_i + c_k_0 - c_j_0, c_k_n - c_j_n, c_k_x - c_j_x)
2802 * with each coefficient (except e_i, c_k_0 and c_j_0)
2803 * represented as a pair of non-negative coefficients.
2805 static int add_inter_constraints(struct isl_sched_graph *graph,
2806 struct isl_sched_edge *edge, __isl_take isl_map *map, int pos)
2808 unsigned total;
2809 isl_ctx *ctx = isl_map_get_ctx(map);
2810 isl_space *dim;
2811 isl_dim_map *dim_map;
2812 isl_basic_set *coef;
2813 struct isl_sched_node *src = edge->src;
2814 struct isl_sched_node *dst = edge->dst;
2816 coef = inter_coefficients(graph, map);
2817 if (!coef)
2818 return -1;
2820 dim = isl_space_domain(isl_space_unwrap(isl_basic_set_get_space(coef)));
2822 total = isl_basic_set_total_dim(graph->lp);
2823 dim_map = isl_dim_map_alloc(ctx, total);
2825 isl_dim_map_range(dim_map, 3 + pos, 0, 0, 0, 1, -1);
2827 isl_dim_map_range(dim_map, dst->start, 0, 0, 0, 1, 1);
2828 isl_dim_map_range(dim_map, dst->start + 1, 2, 1, 1, dst->nparam, -1);
2829 isl_dim_map_range(dim_map, dst->start + 2, 2, 1, 1, dst->nparam, 1);
2830 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 1, 2,
2831 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
2832 dst->nvar, -1);
2833 isl_dim_map_range(dim_map, dst->start + 2 * dst->nparam + 2, 2,
2834 isl_space_dim(dim, isl_dim_set) + src->nvar, 1,
2835 dst->nvar, 1);
2837 isl_dim_map_range(dim_map, src->start, 0, 0, 0, 1, -1);
2838 isl_dim_map_range(dim_map, src->start + 1, 2, 1, 1, src->nparam, 1);
2839 isl_dim_map_range(dim_map, src->start + 2, 2, 1, 1, src->nparam, -1);
2840 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 1, 2,
2841 isl_space_dim(dim, isl_dim_set), 1,
2842 src->nvar, 1);
2843 isl_dim_map_range(dim_map, src->start + 2 * src->nparam + 2, 2,
2844 isl_space_dim(dim, isl_dim_set), 1,
2845 src->nvar, -1);
2847 graph->lp = isl_basic_set_extend_constraints(graph->lp,
2848 coef->n_eq, coef->n_ineq);
2849 graph->lp = isl_basic_set_add_constraints_dim_map(graph->lp,
2850 coef, dim_map);
2851 isl_space_free(dim);
2853 return 0;
2856 /* Add constraints to graph->lp that force all (conditional) validity
2857 * dependences to be respected and attempt to carry them.
2859 static int add_all_constraints(struct isl_sched_graph *graph)
2861 int i, j;
2862 int pos;
2864 pos = 0;
2865 for (i = 0; i < graph->n_edge; ++i) {
2866 struct isl_sched_edge *edge= &graph->edge[i];
2868 if (!edge->validity && !edge->conditional_validity)
2869 continue;
2871 for (j = 0; j < edge->map->n; ++j) {
2872 isl_basic_map *bmap;
2873 isl_map *map;
2875 bmap = isl_basic_map_copy(edge->map->p[j]);
2876 map = isl_map_from_basic_map(bmap);
2878 if (edge->src == edge->dst &&
2879 add_intra_constraints(graph, edge, map, pos) < 0)
2880 return -1;
2881 if (edge->src != edge->dst &&
2882 add_inter_constraints(graph, edge, map, pos) < 0)
2883 return -1;
2884 ++pos;
2888 return 0;
2891 /* Count the number of equality and inequality constraints
2892 * that will be added to the carry_lp problem.
2893 * We count each edge exactly once.
2895 static int count_all_constraints(struct isl_sched_graph *graph,
2896 int *n_eq, int *n_ineq)
2898 int i, j;
2900 *n_eq = *n_ineq = 0;
2901 for (i = 0; i < graph->n_edge; ++i) {
2902 struct isl_sched_edge *edge= &graph->edge[i];
2903 for (j = 0; j < edge->map->n; ++j) {
2904 isl_basic_map *bmap;
2905 isl_map *map;
2907 bmap = isl_basic_map_copy(edge->map->p[j]);
2908 map = isl_map_from_basic_map(bmap);
2910 if (count_map_constraints(graph, edge, map,
2911 n_eq, n_ineq, 1, 0) < 0)
2912 return -1;
2916 return 0;
2919 /* Construct an LP problem for finding schedule coefficients
2920 * such that the schedule carries as many dependences as possible.
2921 * In particular, for each dependence i, we bound the dependence distance
2922 * from below by e_i, with 0 <= e_i <= 1 and then maximize the sum
2923 * of all e_i's. Dependence with e_i = 0 in the solution are simply
2924 * respected, while those with e_i > 0 (in practice e_i = 1) are carried.
2925 * Note that if the dependence relation is a union of basic maps,
2926 * then we have to consider each basic map individually as it may only
2927 * be possible to carry the dependences expressed by some of those
2928 * basic maps and not all off them.
2929 * Below, we consider each of those basic maps as a separate "edge".
2931 * All variables of the LP are non-negative. The actual coefficients
2932 * may be negative, so each coefficient is represented as the difference
2933 * of two non-negative variables. The negative part always appears
2934 * immediately before the positive part.
2935 * Other than that, the variables have the following order
2937 * - sum of (1 - e_i) over all edges
2938 * - sum of positive and negative parts of all c_n coefficients
2939 * (unconstrained when computing non-parametric schedules)
2940 * - sum of positive and negative parts of all c_x coefficients
2941 * - for each edge
2942 * - e_i
2943 * - for each node
2944 * - c_i_0
2945 * - positive and negative parts of c_i_n (if parametric)
2946 * - positive and negative parts of c_i_x
2948 * The constraints are those from the (validity) edges plus three equalities
2949 * to express the sums and n_edge inequalities to express e_i <= 1.
2951 static int setup_carry_lp(isl_ctx *ctx, struct isl_sched_graph *graph)
2953 int i, j;
2954 int k;
2955 isl_space *dim;
2956 unsigned total;
2957 int n_eq, n_ineq;
2958 int n_edge;
2960 n_edge = 0;
2961 for (i = 0; i < graph->n_edge; ++i)
2962 n_edge += graph->edge[i].map->n;
2964 total = 3 + n_edge;
2965 for (i = 0; i < graph->n; ++i) {
2966 struct isl_sched_node *node = &graph->node[graph->sorted[i]];
2967 node->start = total;
2968 total += 1 + 2 * (node->nparam + node->nvar);
2971 if (count_all_constraints(graph, &n_eq, &n_ineq) < 0)
2972 return -1;
2973 if (count_bound_coefficient_constraints(ctx, graph, &n_eq, &n_ineq) < 0)
2974 return -1;
2976 dim = isl_space_set_alloc(ctx, 0, total);
2977 isl_basic_set_free(graph->lp);
2978 n_eq += 3;
2979 n_ineq += n_edge;
2980 graph->lp = isl_basic_set_alloc_space(dim, 0, n_eq, n_ineq);
2981 graph->lp = isl_basic_set_set_rational(graph->lp);
2983 k = isl_basic_set_alloc_equality(graph->lp);
2984 if (k < 0)
2985 return -1;
2986 isl_seq_clr(graph->lp->eq[k], 1 + total);
2987 isl_int_set_si(graph->lp->eq[k][0], -n_edge);
2988 isl_int_set_si(graph->lp->eq[k][1], 1);
2989 for (i = 0; i < n_edge; ++i)
2990 isl_int_set_si(graph->lp->eq[k][4 + i], 1);
2992 k = isl_basic_set_alloc_equality(graph->lp);
2993 if (k < 0)
2994 return -1;
2995 isl_seq_clr(graph->lp->eq[k], 1 + total);
2996 isl_int_set_si(graph->lp->eq[k][2], -1);
2997 for (i = 0; i < graph->n; ++i) {
2998 int pos = 1 + graph->node[i].start + 1;
3000 for (j = 0; j < 2 * graph->node[i].nparam; ++j)
3001 isl_int_set_si(graph->lp->eq[k][pos + j], 1);
3004 k = isl_basic_set_alloc_equality(graph->lp);
3005 if (k < 0)
3006 return -1;
3007 isl_seq_clr(graph->lp->eq[k], 1 + total);
3008 isl_int_set_si(graph->lp->eq[k][3], -1);
3009 for (i = 0; i < graph->n; ++i) {
3010 struct isl_sched_node *node = &graph->node[i];
3011 int pos = 1 + node->start + 1 + 2 * node->nparam;
3013 for (j = 0; j < 2 * node->nvar; ++j)
3014 isl_int_set_si(graph->lp->eq[k][pos + j], 1);
3017 for (i = 0; i < n_edge; ++i) {
3018 k = isl_basic_set_alloc_inequality(graph->lp);
3019 if (k < 0)
3020 return -1;
3021 isl_seq_clr(graph->lp->ineq[k], 1 + total);
3022 isl_int_set_si(graph->lp->ineq[k][4 + i], -1);
3023 isl_int_set_si(graph->lp->ineq[k][0], 1);
3026 if (add_bound_coefficient_constraints(ctx, graph) < 0)
3027 return -1;
3028 if (add_all_constraints(graph) < 0)
3029 return -1;
3031 return 0;
3034 /* If the schedule_split_scaled option is set and if the linear
3035 * parts of the scheduling rows for all nodes in the graphs have
3036 * non-trivial common divisor, then split off the constant term
3037 * from the linear part.
3038 * The constant term is then placed in a separate band and
3039 * the linear part is reduced.
3041 static int split_scaled(isl_ctx *ctx, struct isl_sched_graph *graph)
3043 int i;
3044 int row;
3045 isl_int gcd, gcd_i;
3047 if (!ctx->opt->schedule_split_scaled)
3048 return 0;
3049 if (graph->n <= 1)
3050 return 0;
3052 if (graph->n_total_row >= graph->max_row)
3053 isl_die(ctx, isl_error_internal,
3054 "too many schedule rows", return -1);
3056 isl_int_init(gcd);
3057 isl_int_init(gcd_i);
3059 isl_int_set_si(gcd, 0);
3061 row = isl_mat_rows(graph->node[0].sched) - 1;
3063 for (i = 0; i < graph->n; ++i) {
3064 struct isl_sched_node *node = &graph->node[i];
3065 int cols = isl_mat_cols(node->sched);
3067 isl_seq_gcd(node->sched->row[row] + 1, cols - 1, &gcd_i);
3068 isl_int_gcd(gcd, gcd, gcd_i);
3071 isl_int_clear(gcd_i);
3073 if (isl_int_cmp_si(gcd, 1) <= 0) {
3074 isl_int_clear(gcd);
3075 return 0;
3078 next_band(graph);
3080 for (i = 0; i < graph->n; ++i) {
3081 struct isl_sched_node *node = &graph->node[i];
3083 isl_map_free(node->sched_map);
3084 node->sched_map = NULL;
3085 node->sched = isl_mat_add_zero_rows(node->sched, 1);
3086 if (!node->sched)
3087 goto error;
3088 isl_int_fdiv_r(node->sched->row[row + 1][0],
3089 node->sched->row[row][0], gcd);
3090 isl_int_fdiv_q(node->sched->row[row][0],
3091 node->sched->row[row][0], gcd);
3092 isl_int_mul(node->sched->row[row][0],
3093 node->sched->row[row][0], gcd);
3094 node->sched = isl_mat_scale_down_row(node->sched, row, gcd);
3095 if (!node->sched)
3096 goto error;
3097 node->band[graph->n_total_row] = graph->n_band;
3100 graph->n_total_row++;
3102 isl_int_clear(gcd);
3103 return 0;
3104 error:
3105 isl_int_clear(gcd);
3106 return -1;
3109 static int compute_component_schedule(isl_ctx *ctx,
3110 struct isl_sched_graph *graph);
3112 /* Is the schedule row "sol" trivial on node "node"?
3113 * That is, is the solution zero on the dimensions orthogonal to
3114 * the previously found solutions?
3115 * Return 1 if the solution is trivial, 0 if it is not and -1 on error.
3117 * Each coefficient is represented as the difference between
3118 * two non-negative values in "sol". "sol" has been computed
3119 * in terms of the original iterators (i.e., without use of cmap).
3120 * We construct the schedule row s and write it as a linear
3121 * combination of (linear combinations of) previously computed schedule rows.
3122 * s = Q c or c = U s.
3123 * If the final entries of c are all zero, then the solution is trivial.
3125 static int is_trivial(struct isl_sched_node *node, __isl_keep isl_vec *sol)
3127 int i;
3128 int pos;
3129 int trivial;
3130 isl_ctx *ctx;
3131 isl_vec *node_sol;
3133 if (!sol)
3134 return -1;
3135 if (node->nvar == node->rank)
3136 return 0;
3138 ctx = isl_vec_get_ctx(sol);
3139 node_sol = isl_vec_alloc(ctx, node->nvar);
3140 if (!node_sol)
3141 return -1;
3143 pos = 1 + node->start + 1 + 2 * node->nparam;
3145 for (i = 0; i < node->nvar; ++i)
3146 isl_int_sub(node_sol->el[i],
3147 sol->el[pos + 2 * i + 1], sol->el[pos + 2 * i]);
3149 node_sol = isl_mat_vec_product(isl_mat_copy(node->cinv), node_sol);
3151 if (!node_sol)
3152 return -1;
3154 trivial = isl_seq_first_non_zero(node_sol->el + node->rank,
3155 node->nvar - node->rank) == -1;
3157 isl_vec_free(node_sol);
3159 return trivial;
3162 /* Is the schedule row "sol" trivial on any node where it should
3163 * not be trivial?
3164 * "sol" has been computed in terms of the original iterators
3165 * (i.e., without use of cmap).
3166 * Return 1 if any solution is trivial, 0 if they are not and -1 on error.
3168 static int is_any_trivial(struct isl_sched_graph *graph,
3169 __isl_keep isl_vec *sol)
3171 int i;
3173 for (i = 0; i < graph->n; ++i) {
3174 struct isl_sched_node *node = &graph->node[i];
3175 int trivial;
3177 if (!needs_row(graph, node))
3178 continue;
3179 trivial = is_trivial(node, sol);
3180 if (trivial < 0 || trivial)
3181 return trivial;
3184 return 0;
3187 /* Construct a schedule row for each node such that as many dependences
3188 * as possible are carried and then continue with the next band.
3190 * If the computed schedule row turns out to be trivial on one or
3191 * more nodes where it should not be trivial, then we throw it away
3192 * and try again on each component separately.
3194 static int carry_dependences(isl_ctx *ctx, struct isl_sched_graph *graph)
3196 int i;
3197 int n_edge;
3198 int trivial;
3199 isl_vec *sol;
3200 isl_basic_set *lp;
3202 n_edge = 0;
3203 for (i = 0; i < graph->n_edge; ++i)
3204 n_edge += graph->edge[i].map->n;
3206 if (setup_carry_lp(ctx, graph) < 0)
3207 return -1;
3209 lp = isl_basic_set_copy(graph->lp);
3210 sol = isl_tab_basic_set_non_neg_lexmin(lp);
3211 if (!sol)
3212 return -1;
3214 if (sol->size == 0) {
3215 isl_vec_free(sol);
3216 isl_die(ctx, isl_error_internal,
3217 "error in schedule construction", return -1);
3220 isl_int_divexact(sol->el[1], sol->el[1], sol->el[0]);
3221 if (isl_int_cmp_si(sol->el[1], n_edge) >= 0) {
3222 isl_vec_free(sol);
3223 isl_die(ctx, isl_error_unknown,
3224 "unable to carry dependences", return -1);
3227 trivial = is_any_trivial(graph, sol);
3228 if (trivial < 0) {
3229 sol = isl_vec_free(sol);
3230 } else if (trivial) {
3231 isl_vec_free(sol);
3232 if (graph->scc > 1)
3233 return compute_component_schedule(ctx, graph);
3234 isl_die(ctx, isl_error_unknown,
3235 "unable to construct non-trivial solution", return -1);
3238 if (update_schedule(graph, sol, 0, 0) < 0)
3239 return -1;
3241 if (split_scaled(ctx, graph) < 0)
3242 return -1;
3244 return compute_next_band(ctx, graph);
3247 /* Are there any (non-empty) (conditional) validity edges in the graph?
3249 static int has_validity_edges(struct isl_sched_graph *graph)
3251 int i;
3253 for (i = 0; i < graph->n_edge; ++i) {
3254 int empty;
3256 empty = isl_map_plain_is_empty(graph->edge[i].map);
3257 if (empty < 0)
3258 return -1;
3259 if (empty)
3260 continue;
3261 if (graph->edge[i].validity ||
3262 graph->edge[i].conditional_validity)
3263 return 1;
3266 return 0;
3269 /* Should we apply a Feautrier step?
3270 * That is, did the user request the Feautrier algorithm and are
3271 * there any validity dependences (left)?
3273 static int need_feautrier_step(isl_ctx *ctx, struct isl_sched_graph *graph)
3275 if (ctx->opt->schedule_algorithm != ISL_SCHEDULE_ALGORITHM_FEAUTRIER)
3276 return 0;
3278 return has_validity_edges(graph);
3281 /* Compute a schedule for a connected dependence graph using Feautrier's
3282 * multi-dimensional scheduling algorithm.
3283 * The original algorithm is described in [1].
3284 * The main idea is to minimize the number of scheduling dimensions, by
3285 * trying to satisfy as many dependences as possible per scheduling dimension.
3287 * [1] P. Feautrier, Some Efficient Solutions to the Affine Scheduling
3288 * Problem, Part II: Multi-Dimensional Time.
3289 * In Intl. Journal of Parallel Programming, 1992.
3291 static int compute_schedule_wcc_feautrier(isl_ctx *ctx,
3292 struct isl_sched_graph *graph)
3294 return carry_dependences(ctx, graph);
3297 /* Turn off the "local" bit on all (condition) edges.
3299 static void clear_local_edges(struct isl_sched_graph *graph)
3301 int i;
3303 for (i = 0; i < graph->n_edge; ++i)
3304 if (graph->edge[i].condition)
3305 graph->edge[i].local = 0;
3308 /* Does "graph" have both condition and conditional validity edges?
3310 static int need_condition_check(struct isl_sched_graph *graph)
3312 int i;
3313 int any_condition = 0;
3314 int any_conditional_validity = 0;
3316 for (i = 0; i < graph->n_edge; ++i) {
3317 if (graph->edge[i].condition)
3318 any_condition = 1;
3319 if (graph->edge[i].conditional_validity)
3320 any_conditional_validity = 1;
3323 return any_condition && any_conditional_validity;
3326 /* Does "graph" contain any coincidence edge?
3328 static int has_any_coincidence(struct isl_sched_graph *graph)
3330 int i;
3332 for (i = 0; i < graph->n_edge; ++i)
3333 if (graph->edge[i].coincidence)
3334 return 1;
3336 return 0;
3339 /* Extract the final schedule row as a map with the iteration domain
3340 * of "node" as domain.
3342 static __isl_give isl_map *final_row(struct isl_sched_node *node)
3344 isl_local_space *ls;
3345 isl_aff *aff;
3346 int row;
3348 row = isl_mat_rows(node->sched) - 1;
3349 ls = isl_local_space_from_space(isl_space_copy(node->dim));
3350 aff = extract_schedule_row(ls, node, row);
3351 return isl_map_from_aff(aff);
3354 /* Is the conditional validity dependence in the edge with index "edge_index"
3355 * violated by the latest (i.e., final) row of the schedule?
3356 * That is, is i scheduled after j
3357 * for any conditional validity dependence i -> j?
3359 static int is_violated(struct isl_sched_graph *graph, int edge_index)
3361 isl_map *src_sched, *dst_sched, *map;
3362 struct isl_sched_edge *edge = &graph->edge[edge_index];
3363 int empty;
3365 src_sched = final_row(edge->src);
3366 dst_sched = final_row(edge->dst);
3367 map = isl_map_copy(edge->map);
3368 map = isl_map_apply_domain(map, src_sched);
3369 map = isl_map_apply_range(map, dst_sched);
3370 map = isl_map_order_gt(map, isl_dim_in, 0, isl_dim_out, 0);
3371 empty = isl_map_is_empty(map);
3372 isl_map_free(map);
3374 if (empty < 0)
3375 return -1;
3377 return !empty;
3380 /* Does the domain of "umap" intersect "uset"?
3382 static int domain_intersects(__isl_keep isl_union_map *umap,
3383 __isl_keep isl_union_set *uset)
3385 int empty;
3387 umap = isl_union_map_copy(umap);
3388 umap = isl_union_map_intersect_domain(umap, isl_union_set_copy(uset));
3389 empty = isl_union_map_is_empty(umap);
3390 isl_union_map_free(umap);
3392 return empty < 0 ? -1 : !empty;
3395 /* Does the range of "umap" intersect "uset"?
3397 static int range_intersects(__isl_keep isl_union_map *umap,
3398 __isl_keep isl_union_set *uset)
3400 int empty;
3402 umap = isl_union_map_copy(umap);
3403 umap = isl_union_map_intersect_range(umap, isl_union_set_copy(uset));
3404 empty = isl_union_map_is_empty(umap);
3405 isl_union_map_free(umap);
3407 return empty < 0 ? -1 : !empty;
3410 /* Are the condition dependences of "edge" local with respect to
3411 * the current schedule?
3413 * That is, are domain and range of the condition dependences mapped
3414 * to the same point?
3416 * In other words, is the condition false?
3418 static int is_condition_false(struct isl_sched_edge *edge)
3420 isl_union_map *umap;
3421 isl_map *map, *sched, *test;
3422 int local;
3424 umap = isl_union_map_copy(edge->tagged_condition);
3425 umap = isl_union_map_zip(umap);
3426 umap = isl_union_set_unwrap(isl_union_map_domain(umap));
3427 map = isl_map_from_union_map(umap);
3429 sched = node_extract_schedule(edge->src);
3430 map = isl_map_apply_domain(map, sched);
3431 sched = node_extract_schedule(edge->dst);
3432 map = isl_map_apply_range(map, sched);
3434 test = isl_map_identity(isl_map_get_space(map));
3435 local = isl_map_is_subset(map, test);
3436 isl_map_free(map);
3437 isl_map_free(test);
3439 return local;
3442 /* Does "graph" have any satisfied condition edges that
3443 * are adjacent to the conditional validity constraint with
3444 * domain "conditional_source" and range "conditional_sink"?
3446 * A satisfied condition is one that is not local.
3447 * If a condition was forced to be local already (i.e., marked as local)
3448 * then there is no need to check if it is in fact local.
3450 * Additionally, mark all adjacent condition edges found as local.
3452 static int has_adjacent_true_conditions(struct isl_sched_graph *graph,
3453 __isl_keep isl_union_set *conditional_source,
3454 __isl_keep isl_union_set *conditional_sink)
3456 int i;
3457 int any = 0;
3459 for (i = 0; i < graph->n_edge; ++i) {
3460 int adjacent, local;
3461 isl_union_map *condition;
3463 if (!graph->edge[i].condition)
3464 continue;
3465 if (graph->edge[i].local)
3466 continue;
3468 condition = graph->edge[i].tagged_condition;
3469 adjacent = domain_intersects(condition, conditional_sink);
3470 if (adjacent >= 0 && !adjacent)
3471 adjacent = range_intersects(condition,
3472 conditional_source);
3473 if (adjacent < 0)
3474 return -1;
3475 if (!adjacent)
3476 continue;
3478 graph->edge[i].local = 1;
3480 local = is_condition_false(&graph->edge[i]);
3481 if (local < 0)
3482 return -1;
3483 if (!local)
3484 any = 1;
3487 return any;
3490 /* Are there any violated conditional validity dependences with
3491 * adjacent condition dependences that are not local with respect
3492 * to the current schedule?
3493 * That is, is the conditional validity constraint violated?
3495 * Additionally, mark all those adjacent condition dependences as local.
3496 * We also mark those adjacent condition dependences that were not marked
3497 * as local before, but just happened to be local already. This ensures
3498 * that they remain local if the schedule is recomputed.
3500 * We first collect domain and range of all violated conditional validity
3501 * dependences and then check if there are any adjacent non-local
3502 * condition dependences.
3504 static int has_violated_conditional_constraint(isl_ctx *ctx,
3505 struct isl_sched_graph *graph)
3507 int i;
3508 int any = 0;
3509 isl_union_set *source, *sink;
3511 source = isl_union_set_empty(isl_space_params_alloc(ctx, 0));
3512 sink = isl_union_set_empty(isl_space_params_alloc(ctx, 0));
3513 for (i = 0; i < graph->n_edge; ++i) {
3514 isl_union_set *uset;
3515 isl_union_map *umap;
3516 int violated;
3518 if (!graph->edge[i].conditional_validity)
3519 continue;
3521 violated = is_violated(graph, i);
3522 if (violated < 0)
3523 goto error;
3524 if (!violated)
3525 continue;
3527 any = 1;
3529 umap = isl_union_map_copy(graph->edge[i].tagged_validity);
3530 uset = isl_union_map_domain(umap);
3531 source = isl_union_set_union(source, uset);
3532 source = isl_union_set_coalesce(source);
3534 umap = isl_union_map_copy(graph->edge[i].tagged_validity);
3535 uset = isl_union_map_range(umap);
3536 sink = isl_union_set_union(sink, uset);
3537 sink = isl_union_set_coalesce(sink);
3540 if (any)
3541 any = has_adjacent_true_conditions(graph, source, sink);
3543 isl_union_set_free(source);
3544 isl_union_set_free(sink);
3545 return any;
3546 error:
3547 isl_union_set_free(source);
3548 isl_union_set_free(sink);
3549 return -1;
3552 /* Compute a schedule for a connected dependence graph.
3553 * We try to find a sequence of as many schedule rows as possible that result
3554 * in non-negative dependence distances (independent of the previous rows
3555 * in the sequence, i.e., such that the sequence is tilable), with as
3556 * many of the initial rows as possible satisfying the coincidence constraints.
3557 * If we can't find any more rows we either
3558 * - split between SCCs and start over (assuming we found an interesting
3559 * pair of SCCs between which to split)
3560 * - continue with the next band (assuming the current band has at least
3561 * one row)
3562 * - try to carry as many dependences as possible and continue with the next
3563 * band
3565 * If Feautrier's algorithm is selected, we first recursively try to satisfy
3566 * as many validity dependences as possible. When all validity dependences
3567 * are satisfied we extend the schedule to a full-dimensional schedule.
3569 * If we manage to complete the schedule, we finish off by topologically
3570 * sorting the statements based on the remaining dependences.
3572 * If ctx->opt->schedule_outer_coincidence is set, then we force the
3573 * outermost dimension to satisfy the coincidence constraints. If this
3574 * turns out to be impossible, we fall back on the general scheme above
3575 * and try to carry as many dependences as possible.
3577 * If "graph" contains both condition and conditional validity dependences,
3578 * then we need to check that that the conditional schedule constraint
3579 * is satisfied, i.e., there are no violated conditional validity dependences
3580 * that are adjacent to any non-local condition dependences.
3581 * If there are, then we mark all those adjacent condition dependences
3582 * as local and recompute the current band. Those dependences that
3583 * are marked local will then be forced to be local.
3584 * The initial computation is performed with no dependences marked as local.
3585 * If we are lucky, then there will be no violated conditional validity
3586 * dependences adjacent to any non-local condition dependences.
3587 * Otherwise, we mark some additional condition dependences as local and
3588 * recompute. We continue this process until there are no violations left or
3589 * until we are no longer able to compute a schedule.
3590 * Since there are only a finite number of dependences,
3591 * there will only be a finite number of iterations.
3593 static int compute_schedule_wcc(isl_ctx *ctx, struct isl_sched_graph *graph)
3595 int has_coincidence;
3596 int use_coincidence;
3597 int force_coincidence = 0;
3598 int check_conditional;
3600 if (detect_sccs(ctx, graph) < 0)
3601 return -1;
3602 if (sort_sccs(graph) < 0)
3603 return -1;
3605 if (compute_maxvar(graph) < 0)
3606 return -1;
3608 if (need_feautrier_step(ctx, graph))
3609 return compute_schedule_wcc_feautrier(ctx, graph);
3611 clear_local_edges(graph);
3612 check_conditional = need_condition_check(graph);
3613 has_coincidence = has_any_coincidence(graph);
3615 if (ctx->opt->schedule_outer_coincidence)
3616 force_coincidence = 1;
3618 use_coincidence = has_coincidence;
3619 while (graph->n_row < graph->maxvar) {
3620 isl_vec *sol;
3621 int violated;
3622 int coincident;
3624 graph->src_scc = -1;
3625 graph->dst_scc = -1;
3627 if (setup_lp(ctx, graph, use_coincidence) < 0)
3628 return -1;
3629 sol = solve_lp(graph);
3630 if (!sol)
3631 return -1;
3632 if (sol->size == 0) {
3633 int empty = graph->n_total_row == graph->band_start;
3635 isl_vec_free(sol);
3636 if (use_coincidence && (!force_coincidence || !empty)) {
3637 use_coincidence = 0;
3638 continue;
3640 if (!ctx->opt->schedule_maximize_band_depth && !empty)
3641 return compute_next_band(ctx, graph);
3642 if (graph->src_scc >= 0)
3643 return compute_split_schedule(ctx, graph);
3644 if (!empty)
3645 return compute_next_band(ctx, graph);
3646 return carry_dependences(ctx, graph);
3648 coincident = !has_coincidence || use_coincidence;
3649 if (update_schedule(graph, sol, 1, coincident) < 0)
3650 return -1;
3652 if (!check_conditional)
3653 continue;
3654 violated = has_violated_conditional_constraint(ctx, graph);
3655 if (violated < 0)
3656 return -1;
3657 if (!violated)
3658 continue;
3659 if (reset_band(graph) < 0)
3660 return -1;
3661 use_coincidence = has_coincidence;
3664 if (graph->n_total_row > graph->band_start)
3665 next_band(graph);
3666 return sort_statements(ctx, graph);
3669 /* Add a row to the schedules that separates the SCCs and move
3670 * to the next band.
3672 static int split_on_scc(isl_ctx *ctx, struct isl_sched_graph *graph)
3674 int i;
3676 if (graph->n_total_row >= graph->max_row)
3677 isl_die(ctx, isl_error_internal,
3678 "too many schedule rows", return -1);
3680 for (i = 0; i < graph->n; ++i) {
3681 struct isl_sched_node *node = &graph->node[i];
3682 int row = isl_mat_rows(node->sched);
3684 isl_map_free(node->sched_map);
3685 node->sched_map = NULL;
3686 node->sched = isl_mat_add_zero_rows(node->sched, 1);
3687 node->sched = isl_mat_set_element_si(node->sched, row, 0,
3688 node->scc);
3689 if (!node->sched)
3690 return -1;
3691 node->band[graph->n_total_row] = graph->n_band;
3694 graph->n_total_row++;
3695 next_band(graph);
3697 return 0;
3700 /* Compute a schedule for each component (identified by node->scc)
3701 * of the dependence graph separately and then combine the results.
3702 * Depending on the setting of schedule_fuse, a component may be
3703 * either weakly or strongly connected.
3705 * The band_id is adjusted such that each component has a separate id.
3706 * Note that the band_id may have already been set to a value different
3707 * from zero by compute_split_schedule.
3709 static int compute_component_schedule(isl_ctx *ctx,
3710 struct isl_sched_graph *graph)
3712 int wcc, i;
3713 int n, n_edge;
3714 int n_total_row, orig_total_row;
3715 int n_band, orig_band;
3717 if (ctx->opt->schedule_fuse == ISL_SCHEDULE_FUSE_MIN ||
3718 ctx->opt->schedule_separate_components)
3719 if (split_on_scc(ctx, graph) < 0)
3720 return -1;
3722 n_total_row = 0;
3723 orig_total_row = graph->n_total_row;
3724 n_band = 0;
3725 orig_band = graph->n_band;
3726 for (i = 0; i < graph->n; ++i)
3727 graph->node[i].band_id[graph->n_band] += graph->node[i].scc;
3728 for (wcc = 0; wcc < graph->scc; ++wcc) {
3729 n = 0;
3730 for (i = 0; i < graph->n; ++i)
3731 if (graph->node[i].scc == wcc)
3732 n++;
3733 n_edge = 0;
3734 for (i = 0; i < graph->n_edge; ++i)
3735 if (graph->edge[i].src->scc == wcc &&
3736 graph->edge[i].dst->scc == wcc)
3737 n_edge++;
3739 if (compute_sub_schedule(ctx, graph, n, n_edge,
3740 &node_scc_exactly,
3741 &edge_scc_exactly, wcc, 1) < 0)
3742 return -1;
3743 if (graph->n_total_row > n_total_row)
3744 n_total_row = graph->n_total_row;
3745 graph->n_total_row = orig_total_row;
3746 if (graph->n_band > n_band)
3747 n_band = graph->n_band;
3748 graph->n_band = orig_band;
3751 graph->n_total_row = n_total_row;
3752 graph->n_band = n_band;
3754 return pad_schedule(graph);
3757 /* Compute a schedule for the given dependence graph.
3758 * We first check if the graph is connected (through validity and conditional
3759 * validity dependences) and, if not, compute a schedule
3760 * for each component separately.
3761 * If schedule_fuse is set to minimal fusion, then we check for strongly
3762 * connected components instead and compute a separate schedule for
3763 * each such strongly connected component.
3765 static int compute_schedule(isl_ctx *ctx, struct isl_sched_graph *graph)
3767 if (ctx->opt->schedule_fuse == ISL_SCHEDULE_FUSE_MIN) {
3768 if (detect_sccs(ctx, graph) < 0)
3769 return -1;
3770 } else {
3771 if (detect_wccs(ctx, graph) < 0)
3772 return -1;
3775 if (graph->scc > 1)
3776 return compute_component_schedule(ctx, graph);
3778 return compute_schedule_wcc(ctx, graph);
3781 /* Compute a schedule on sc->domain that respects the given schedule
3782 * constraints.
3784 * In particular, the schedule respects all the validity dependences.
3785 * If the default isl scheduling algorithm is used, it tries to minimize
3786 * the dependence distances over the proximity dependences.
3787 * If Feautrier's scheduling algorithm is used, the proximity dependence
3788 * distances are only minimized during the extension to a full-dimensional
3789 * schedule.
3791 * If there are any condition and conditional validity dependences,
3792 * then the conditional validity dependences may be violated inside
3793 * a tilable band, provided they have no adjacent non-local
3794 * condition dependences.
3796 __isl_give isl_schedule *isl_schedule_constraints_compute_schedule(
3797 __isl_take isl_schedule_constraints *sc)
3799 isl_ctx *ctx = isl_schedule_constraints_get_ctx(sc);
3800 struct isl_sched_graph graph = { 0 };
3801 isl_schedule *sched;
3802 struct isl_extract_edge_data data;
3803 enum isl_edge_type i;
3805 sc = isl_schedule_constraints_align_params(sc);
3806 if (!sc)
3807 return NULL;
3809 graph.n = isl_union_set_n_set(sc->domain);
3810 if (graph.n == 0)
3811 goto empty;
3812 if (graph_alloc(ctx, &graph, graph.n,
3813 isl_schedule_constraints_n_map(sc)) < 0)
3814 goto error;
3815 if (compute_max_row(&graph, sc->domain) < 0)
3816 goto error;
3817 graph.root = 1;
3818 graph.n = 0;
3819 if (isl_union_set_foreach_set(sc->domain, &extract_node, &graph) < 0)
3820 goto error;
3821 if (graph_init_table(ctx, &graph) < 0)
3822 goto error;
3823 for (i = isl_edge_first; i <= isl_edge_last; ++i)
3824 graph.max_edge[i] = isl_union_map_n_map(sc->constraint[i]);
3825 if (graph_init_edge_tables(ctx, &graph) < 0)
3826 goto error;
3827 graph.n_edge = 0;
3828 data.graph = &graph;
3829 for (i = isl_edge_first; i <= isl_edge_last; ++i) {
3830 data.type = i;
3831 if (isl_union_map_foreach_map(sc->constraint[i],
3832 &extract_edge, &data) < 0)
3833 goto error;
3836 if (compute_schedule(ctx, &graph) < 0)
3837 goto error;
3839 empty:
3840 sched = extract_schedule(&graph, isl_union_set_get_space(sc->domain));
3842 graph_free(ctx, &graph);
3843 isl_schedule_constraints_free(sc);
3845 return sched;
3846 error:
3847 graph_free(ctx, &graph);
3848 isl_schedule_constraints_free(sc);
3849 return NULL;
3852 /* Compute a schedule for the given union of domains that respects
3853 * all the validity dependences and minimizes
3854 * the dependence distances over the proximity dependences.
3856 * This function is kept for backward compatibility.
3858 __isl_give isl_schedule *isl_union_set_compute_schedule(
3859 __isl_take isl_union_set *domain,
3860 __isl_take isl_union_map *validity,
3861 __isl_take isl_union_map *proximity)
3863 isl_schedule_constraints *sc;
3865 sc = isl_schedule_constraints_on_domain(domain);
3866 sc = isl_schedule_constraints_set_validity(sc, validity);
3867 sc = isl_schedule_constraints_set_proximity(sc, proximity);
3869 return isl_schedule_constraints_compute_schedule(sc);
3872 void *isl_schedule_free(__isl_take isl_schedule *sched)
3874 int i;
3875 if (!sched)
3876 return NULL;
3878 if (--sched->ref > 0)
3879 return NULL;
3881 for (i = 0; i < sched->n; ++i) {
3882 isl_multi_aff_free(sched->node[i].sched);
3883 free(sched->node[i].band_end);
3884 free(sched->node[i].band_id);
3885 free(sched->node[i].coincident);
3887 isl_space_free(sched->dim);
3888 isl_band_list_free(sched->band_forest);
3889 free(sched);
3890 return NULL;
3893 isl_ctx *isl_schedule_get_ctx(__isl_keep isl_schedule *schedule)
3895 return schedule ? isl_space_get_ctx(schedule->dim) : NULL;
3898 /* Set max_out to the maximal number of output dimensions over
3899 * all maps.
3901 static int update_max_out(__isl_take isl_map *map, void *user)
3903 int *max_out = user;
3904 int n_out = isl_map_dim(map, isl_dim_out);
3906 if (n_out > *max_out)
3907 *max_out = n_out;
3909 isl_map_free(map);
3910 return 0;
3913 /* Internal data structure for map_pad_range.
3915 * "max_out" is the maximal schedule dimension.
3916 * "res" collects the results.
3918 struct isl_pad_schedule_map_data {
3919 int max_out;
3920 isl_union_map *res;
3923 /* Pad the range of the given map with zeros to data->max_out and
3924 * then add the result to data->res.
3926 static int map_pad_range(__isl_take isl_map *map, void *user)
3928 struct isl_pad_schedule_map_data *data = user;
3929 int i;
3930 int n_out = isl_map_dim(map, isl_dim_out);
3932 map = isl_map_add_dims(map, isl_dim_out, data->max_out - n_out);
3933 for (i = n_out; i < data->max_out; ++i)
3934 map = isl_map_fix_si(map, isl_dim_out, i, 0);
3936 data->res = isl_union_map_add_map(data->res, map);
3937 if (!data->res)
3938 return -1;
3940 return 0;
3943 /* Pad the ranges of the maps in the union map with zeros such they all have
3944 * the same dimension.
3946 static __isl_give isl_union_map *pad_schedule_map(
3947 __isl_take isl_union_map *umap)
3949 struct isl_pad_schedule_map_data data;
3951 if (!umap)
3952 return NULL;
3953 if (isl_union_map_n_map(umap) <= 1)
3954 return umap;
3956 data.max_out = 0;
3957 if (isl_union_map_foreach_map(umap, &update_max_out, &data.max_out) < 0)
3958 return isl_union_map_free(umap);
3960 data.res = isl_union_map_empty(isl_union_map_get_space(umap));
3961 if (isl_union_map_foreach_map(umap, &map_pad_range, &data) < 0)
3962 data.res = isl_union_map_free(data.res);
3964 isl_union_map_free(umap);
3965 return data.res;
3968 /* Return an isl_union_map of the schedule. If we have already constructed
3969 * a band forest, then this band forest may have been modified so we need
3970 * to extract the isl_union_map from the forest rather than from
3971 * the originally computed schedule. This reconstructed schedule map
3972 * then needs to be padded with zeros to unify the schedule space
3973 * since the result of isl_band_list_get_suffix_schedule may not have
3974 * a unified schedule space.
3976 __isl_give isl_union_map *isl_schedule_get_map(__isl_keep isl_schedule *sched)
3978 int i;
3979 isl_union_map *umap;
3981 if (!sched)
3982 return NULL;
3984 if (sched->band_forest) {
3985 umap = isl_band_list_get_suffix_schedule(sched->band_forest);
3986 return pad_schedule_map(umap);
3989 umap = isl_union_map_empty(isl_space_copy(sched->dim));
3990 for (i = 0; i < sched->n; ++i) {
3991 isl_multi_aff *ma;
3993 ma = isl_multi_aff_copy(sched->node[i].sched);
3994 umap = isl_union_map_add_map(umap, isl_map_from_multi_aff(ma));
3997 return umap;
4000 static __isl_give isl_band_list *construct_band_list(
4001 __isl_keep isl_schedule *schedule, __isl_keep isl_band *parent,
4002 int band_nr, int *parent_active, int n_active);
4004 /* Construct an isl_band structure for the band in the given schedule
4005 * with sequence number band_nr for the n_active nodes marked by active.
4006 * If the nodes don't have a band with the given sequence number,
4007 * then a band without members is created.
4009 * Because of the way the schedule is constructed, we know that
4010 * the position of the band inside the schedule of a node is the same
4011 * for all active nodes.
4013 * The partial schedule for the band is created before the children
4014 * are created to that construct_band_list can refer to the partial
4015 * schedule of the parent.
4017 static __isl_give isl_band *construct_band(__isl_keep isl_schedule *schedule,
4018 __isl_keep isl_band *parent,
4019 int band_nr, int *active, int n_active)
4021 int i, j;
4022 isl_ctx *ctx = isl_schedule_get_ctx(schedule);
4023 isl_band *band;
4024 unsigned start, end;
4026 band = isl_band_alloc(ctx);
4027 if (!band)
4028 return NULL;
4030 band->schedule = schedule;
4031 band->parent = parent;
4033 for (i = 0; i < schedule->n; ++i)
4034 if (active[i])
4035 break;
4037 if (i >= schedule->n)
4038 isl_die(ctx, isl_error_internal,
4039 "band without active statements", goto error);
4041 start = band_nr ? schedule->node[i].band_end[band_nr - 1] : 0;
4042 end = band_nr < schedule->node[i].n_band ?
4043 schedule->node[i].band_end[band_nr] : start;
4044 band->n = end - start;
4046 band->coincident = isl_alloc_array(ctx, int, band->n);
4047 if (band->n && !band->coincident)
4048 goto error;
4050 for (j = 0; j < band->n; ++j)
4051 band->coincident[j] = schedule->node[i].coincident[start + j];
4053 band->pma = isl_union_pw_multi_aff_empty(isl_space_copy(schedule->dim));
4054 for (i = 0; i < schedule->n; ++i) {
4055 isl_multi_aff *ma;
4056 isl_pw_multi_aff *pma;
4057 unsigned n_out;
4059 if (!active[i])
4060 continue;
4062 ma = isl_multi_aff_copy(schedule->node[i].sched);
4063 n_out = isl_multi_aff_dim(ma, isl_dim_out);
4064 ma = isl_multi_aff_drop_dims(ma, isl_dim_out, end, n_out - end);
4065 ma = isl_multi_aff_drop_dims(ma, isl_dim_out, 0, start);
4066 pma = isl_pw_multi_aff_from_multi_aff(ma);
4067 band->pma = isl_union_pw_multi_aff_add_pw_multi_aff(band->pma,
4068 pma);
4070 if (!band->pma)
4071 goto error;
4073 for (i = 0; i < schedule->n; ++i)
4074 if (active[i] && schedule->node[i].n_band > band_nr + 1)
4075 break;
4077 if (i < schedule->n) {
4078 band->children = construct_band_list(schedule, band,
4079 band_nr + 1, active, n_active);
4080 if (!band->children)
4081 goto error;
4084 return band;
4085 error:
4086 isl_band_free(band);
4087 return NULL;
4090 /* Internal data structure used inside cmp_band and pw_multi_aff_extract_int.
4092 * r is set to a negative value if anything goes wrong.
4094 * c1 stores the result of extract_int.
4095 * c2 is a temporary value used inside cmp_band_in_ancestor.
4096 * t is a temporary value used inside extract_int.
4098 * first and equal are used inside extract_int.
4099 * first is set if we are looking at the first isl_multi_aff inside
4100 * the isl_union_pw_multi_aff.
4101 * equal is set if all the isl_multi_affs have been equal so far.
4103 struct isl_cmp_band_data {
4104 int r;
4106 int first;
4107 int equal;
4109 isl_int t;
4110 isl_int c1;
4111 isl_int c2;
4114 /* Check if "ma" assigns a constant value.
4115 * Note that this function is only called on isl_multi_affs
4116 * with a single output dimension.
4118 * If "ma" assigns a constant value then we compare it to data->c1
4119 * or assign it to data->c1 if this is the first isl_multi_aff we consider.
4120 * If "ma" does not assign a constant value or if it assigns a value
4121 * that is different from data->c1, then we set data->equal to zero
4122 * and terminate the check.
4124 static int multi_aff_extract_int(__isl_take isl_set *set,
4125 __isl_take isl_multi_aff *ma, void *user)
4127 isl_aff *aff;
4128 struct isl_cmp_band_data *data = user;
4130 aff = isl_multi_aff_get_aff(ma, 0);
4131 data->r = isl_aff_is_cst(aff);
4132 if (data->r >= 0 && data->r) {
4133 isl_aff_get_constant(aff, &data->t);
4134 if (data->first) {
4135 isl_int_set(data->c1, data->t);
4136 data->first = 0;
4137 } else if (!isl_int_eq(data->c1, data->t))
4138 data->equal = 0;
4139 } else if (data->r >= 0 && !data->r)
4140 data->equal = 0;
4142 isl_aff_free(aff);
4143 isl_set_free(set);
4144 isl_multi_aff_free(ma);
4146 if (data->r < 0)
4147 return -1;
4148 if (!data->equal)
4149 return -1;
4150 return 0;
4153 /* This function is called for each isl_pw_multi_aff in
4154 * the isl_union_pw_multi_aff checked by extract_int.
4155 * Check all the isl_multi_affs inside "pma".
4157 static int pw_multi_aff_extract_int(__isl_take isl_pw_multi_aff *pma,
4158 void *user)
4160 int r;
4162 r = isl_pw_multi_aff_foreach_piece(pma, &multi_aff_extract_int, user);
4163 isl_pw_multi_aff_free(pma);
4165 return r;
4168 /* Check if "upma" assigns a single constant value to its domain.
4169 * If so, return 1 and store the result in data->c1.
4170 * If not, return 0.
4172 * A negative return value from isl_union_pw_multi_aff_foreach_pw_multi_aff
4173 * means that either an error occurred or that we have broken off the check
4174 * because we already know the result is going to be negative.
4175 * In the latter case, data->equal is set to zero.
4177 static int extract_int(__isl_keep isl_union_pw_multi_aff *upma,
4178 struct isl_cmp_band_data *data)
4180 data->first = 1;
4181 data->equal = 1;
4183 if (isl_union_pw_multi_aff_foreach_pw_multi_aff(upma,
4184 &pw_multi_aff_extract_int, data) < 0) {
4185 if (!data->equal)
4186 return 0;
4187 return -1;
4190 return !data->first && data->equal;
4193 /* Compare "b1" and "b2" based on the parent schedule of their ancestor
4194 * "ancestor".
4196 * If the parent of "ancestor" also has a single member, then we
4197 * first try to compare the two band based on the partial schedule
4198 * of this parent.
4200 * Otherwise, or if the result is inconclusive, we look at the partial schedule
4201 * of "ancestor" itself.
4202 * In particular, we specialize the parent schedule based
4203 * on the domains of the child schedules, check if both assign
4204 * a single constant value and, if so, compare the two constant values.
4205 * If the specialized parent schedules do not assign a constant value,
4206 * then they cannot be used to order the two bands and so in this case
4207 * we return 0.
4209 static int cmp_band_in_ancestor(__isl_keep isl_band *b1,
4210 __isl_keep isl_band *b2, struct isl_cmp_band_data *data,
4211 __isl_keep isl_band *ancestor)
4213 isl_union_pw_multi_aff *upma;
4214 isl_union_set *domain;
4215 int r;
4217 if (data->r < 0)
4218 return 0;
4220 if (ancestor->parent && ancestor->parent->n == 1) {
4221 r = cmp_band_in_ancestor(b1, b2, data, ancestor->parent);
4222 if (data->r < 0)
4223 return 0;
4224 if (r)
4225 return r;
4228 upma = isl_union_pw_multi_aff_copy(b1->pma);
4229 domain = isl_union_pw_multi_aff_domain(upma);
4230 upma = isl_union_pw_multi_aff_copy(ancestor->pma);
4231 upma = isl_union_pw_multi_aff_intersect_domain(upma, domain);
4232 r = extract_int(upma, data);
4233 isl_union_pw_multi_aff_free(upma);
4235 if (r < 0)
4236 data->r = -1;
4237 if (r < 0 || !r)
4238 return 0;
4240 isl_int_set(data->c2, data->c1);
4242 upma = isl_union_pw_multi_aff_copy(b2->pma);
4243 domain = isl_union_pw_multi_aff_domain(upma);
4244 upma = isl_union_pw_multi_aff_copy(ancestor->pma);
4245 upma = isl_union_pw_multi_aff_intersect_domain(upma, domain);
4246 r = extract_int(upma, data);
4247 isl_union_pw_multi_aff_free(upma);
4249 if (r < 0)
4250 data->r = -1;
4251 if (r < 0 || !r)
4252 return 0;
4254 return isl_int_cmp(data->c2, data->c1);
4257 /* Compare "a" and "b" based on the parent schedule of their parent.
4259 static int cmp_band(const void *a, const void *b, void *user)
4261 isl_band *b1 = *(isl_band * const *) a;
4262 isl_band *b2 = *(isl_band * const *) b;
4263 struct isl_cmp_band_data *data = user;
4265 return cmp_band_in_ancestor(b1, b2, data, b1->parent);
4268 /* Sort the elements in "list" based on the partial schedules of its parent
4269 * (and ancestors). In particular if the parent assigns constant values
4270 * to the domains of the bands in "list", then the elements are sorted
4271 * according to that order.
4272 * This order should be a more "natural" order for the user, but otherwise
4273 * shouldn't have any effect.
4274 * If we would be constructing an isl_band forest directly in
4275 * isl_schedule_constraints_compute_schedule then there wouldn't be any need
4276 * for a reordering, since the children would be added to the list
4277 * in their natural order automatically.
4279 * If there is only one element in the list, then there is no need to sort
4280 * anything.
4281 * If the partial schedule of the parent has more than one member
4282 * (or if there is no parent), then it's
4283 * defnitely not assigning constant values to the different children in
4284 * the list and so we wouldn't be able to use it to sort the list.
4286 static __isl_give isl_band_list *sort_band_list(__isl_take isl_band_list *list,
4287 __isl_keep isl_band *parent)
4289 struct isl_cmp_band_data data;
4291 if (!list)
4292 return NULL;
4293 if (list->n <= 1)
4294 return list;
4295 if (!parent || parent->n != 1)
4296 return list;
4298 data.r = 0;
4299 isl_int_init(data.c1);
4300 isl_int_init(data.c2);
4301 isl_int_init(data.t);
4302 isl_sort(list->p, list->n, sizeof(list->p[0]), &cmp_band, &data);
4303 if (data.r < 0)
4304 list = isl_band_list_free(list);
4305 isl_int_clear(data.c1);
4306 isl_int_clear(data.c2);
4307 isl_int_clear(data.t);
4309 return list;
4312 /* Construct a list of bands that start at the same position (with
4313 * sequence number band_nr) in the schedules of the nodes that
4314 * were active in the parent band.
4316 * A separate isl_band structure is created for each band_id
4317 * and for each node that does not have a band with sequence
4318 * number band_nr. In the latter case, a band without members
4319 * is created.
4320 * This ensures that if a band has any children, then each node
4321 * that was active in the band is active in exactly one of the children.
4323 static __isl_give isl_band_list *construct_band_list(
4324 __isl_keep isl_schedule *schedule, __isl_keep isl_band *parent,
4325 int band_nr, int *parent_active, int n_active)
4327 int i, j;
4328 isl_ctx *ctx = isl_schedule_get_ctx(schedule);
4329 int *active;
4330 int n_band;
4331 isl_band_list *list;
4333 n_band = 0;
4334 for (i = 0; i < n_active; ++i) {
4335 for (j = 0; j < schedule->n; ++j) {
4336 if (!parent_active[j])
4337 continue;
4338 if (schedule->node[j].n_band <= band_nr)
4339 continue;
4340 if (schedule->node[j].band_id[band_nr] == i) {
4341 n_band++;
4342 break;
4346 for (j = 0; j < schedule->n; ++j)
4347 if (schedule->node[j].n_band <= band_nr)
4348 n_band++;
4350 if (n_band == 1) {
4351 isl_band *band;
4352 list = isl_band_list_alloc(ctx, n_band);
4353 band = construct_band(schedule, parent, band_nr,
4354 parent_active, n_active);
4355 return isl_band_list_add(list, band);
4358 active = isl_alloc_array(ctx, int, schedule->n);
4359 if (schedule->n && !active)
4360 return NULL;
4362 list = isl_band_list_alloc(ctx, n_band);
4364 for (i = 0; i < n_active; ++i) {
4365 int n = 0;
4366 isl_band *band;
4368 for (j = 0; j < schedule->n; ++j) {
4369 active[j] = parent_active[j] &&
4370 schedule->node[j].n_band > band_nr &&
4371 schedule->node[j].band_id[band_nr] == i;
4372 if (active[j])
4373 n++;
4375 if (n == 0)
4376 continue;
4378 band = construct_band(schedule, parent, band_nr, active, n);
4380 list = isl_band_list_add(list, band);
4382 for (i = 0; i < schedule->n; ++i) {
4383 isl_band *band;
4384 if (!parent_active[i])
4385 continue;
4386 if (schedule->node[i].n_band > band_nr)
4387 continue;
4388 for (j = 0; j < schedule->n; ++j)
4389 active[j] = j == i;
4390 band = construct_band(schedule, parent, band_nr, active, 1);
4391 list = isl_band_list_add(list, band);
4394 free(active);
4396 list = sort_band_list(list, parent);
4398 return list;
4401 /* Construct a band forest representation of the schedule and
4402 * return the list of roots.
4404 static __isl_give isl_band_list *construct_forest(
4405 __isl_keep isl_schedule *schedule)
4407 int i;
4408 isl_ctx *ctx = isl_schedule_get_ctx(schedule);
4409 isl_band_list *forest;
4410 int *active;
4412 active = isl_alloc_array(ctx, int, schedule->n);
4413 if (schedule->n && !active)
4414 return NULL;
4416 for (i = 0; i < schedule->n; ++i)
4417 active[i] = 1;
4419 forest = construct_band_list(schedule, NULL, 0, active, schedule->n);
4421 free(active);
4423 return forest;
4426 /* Return the roots of a band forest representation of the schedule.
4428 __isl_give isl_band_list *isl_schedule_get_band_forest(
4429 __isl_keep isl_schedule *schedule)
4431 if (!schedule)
4432 return NULL;
4433 if (!schedule->band_forest)
4434 schedule->band_forest = construct_forest(schedule);
4435 return isl_band_list_dup(schedule->band_forest);
4438 /* Call "fn" on each band in the schedule in depth-first post-order.
4440 int isl_schedule_foreach_band(__isl_keep isl_schedule *sched,
4441 int (*fn)(__isl_keep isl_band *band, void *user), void *user)
4443 int r;
4444 isl_band_list *forest;
4446 if (!sched)
4447 return -1;
4449 forest = isl_schedule_get_band_forest(sched);
4450 r = isl_band_list_foreach_band(forest, fn, user);
4451 isl_band_list_free(forest);
4453 return r;
4456 static __isl_give isl_printer *print_band_list(__isl_take isl_printer *p,
4457 __isl_keep isl_band_list *list);
4459 static __isl_give isl_printer *print_band(__isl_take isl_printer *p,
4460 __isl_keep isl_band *band)
4462 isl_band_list *children;
4464 p = isl_printer_start_line(p);
4465 p = isl_printer_print_union_pw_multi_aff(p, band->pma);
4466 p = isl_printer_end_line(p);
4468 if (!isl_band_has_children(band))
4469 return p;
4471 children = isl_band_get_children(band);
4473 p = isl_printer_indent(p, 4);
4474 p = print_band_list(p, children);
4475 p = isl_printer_indent(p, -4);
4477 isl_band_list_free(children);
4479 return p;
4482 static __isl_give isl_printer *print_band_list(__isl_take isl_printer *p,
4483 __isl_keep isl_band_list *list)
4485 int i, n;
4487 n = isl_band_list_n_band(list);
4488 for (i = 0; i < n; ++i) {
4489 isl_band *band;
4490 band = isl_band_list_get_band(list, i);
4491 p = print_band(p, band);
4492 isl_band_free(band);
4495 return p;
4498 __isl_give isl_printer *isl_printer_print_schedule(__isl_take isl_printer *p,
4499 __isl_keep isl_schedule *schedule)
4501 isl_band_list *forest;
4503 forest = isl_schedule_get_band_forest(schedule);
4505 p = print_band_list(p, forest);
4507 isl_band_list_free(forest);
4509 return p;
4512 void isl_schedule_dump(__isl_keep isl_schedule *schedule)
4514 isl_printer *printer;
4516 if (!schedule)
4517 return;
4519 printer = isl_printer_to_file(isl_schedule_get_ctx(schedule), stderr);
4520 printer = isl_printer_print_schedule(printer, schedule);
4522 isl_printer_free(printer);