hphp/hack/src/parser/full_fidelity_positioned_token.ml

   1 (**
   2  * Copyright (c) 2016, Facebook, Inc.
   3  * All rights reserved.
   4  *
   5  * This source code is licensed under the BSD-style license found in the
   6  * LICENSE file in the "hack" directory of this source tree. An additional grant
   7  * of patent rights can be found in the PATENTS file in the same directory.
   8  *
   9  *)
  10
  11 (**
  12  * Positioned token
  13  *
  14  * A positioned token stores the original source text,
  15  * the offset of the leading trivia, the width of the leading trivia,
  16  * node proper, and trailing trivia. From all this information we can
  17  * rapidly compute the absolute position of any portion of the node,
  18  * or the text.
  19  *
  20  *)
  21
  22 module MinimalToken = Full_fidelity_minimal_token
  23 module Trivia = Full_fidelity_positioned_trivia
  24 module SourceText = Full_fidelity_source_text
  25 module TokenKind = Full_fidelity_token_kind
  26
  27 type t = {
  28   kind: TokenKind.t;
  29   source_text: SourceText.t;
  30   offset: int; (* Beginning of first trivia *)
  31   leading_width: int;
  32   width: int; (* Width of actual token, not counting trivia *)
  33   trailing_width: int;
  34   leading: Trivia.t list;
  35   trailing: Trivia.t list
  36 }
  37
  38 let make kind source_text offset width leading trailing =
  39   let folder sum trivia =
  40     sum + (Trivia.width trivia) in
  41   let leading_width = List.fold_left folder 0 leading in
  42   let trailing_width = List.fold_left folder 0 trailing in
  43   { kind; source_text; offset; leading_width; width; trailing_width;
  44     leading; trailing }
  45
  46 let kind token =
  47   token.kind
  48
  49 let with_kind token kind =
  50   { token with kind }
  51
  52 let source_text token =
  53   token.source_text
  54
  55 let leading_width token =
  56   token.leading_width
  57
  58 let width token =
  59   token.width
  60
  61 let trailing_width token =
  62   token.trailing_width
  63
  64 let full_width token =
  65   (leading_width token) + (width token) + (trailing_width token)
  66
  67 let leading token =
  68   token.leading
  69
  70 let with_leading leading token =
  71   { token with leading }
  72
  73 let trailing token =
  74   token.trailing
  75
  76 let with_trailing trailing token =
  77   { token with trailing }
  78
  79 let has_trivia_kind token kind =
  80   List.exists (fun t -> Trivia.kind t = kind) token.leading ||
  81   List.exists (fun t -> Trivia.kind t = kind) token.trailing
  82
  83 let leading_start_offset token =
  84   token.offset
  85
  86 let leading_end_offset token =
  87   let w = (leading_width token) - 1 in
  88   let w = if w < 0 then 0 else w in
  89   (leading_start_offset token) + w
  90
  91 let start_offset token =
  92   (leading_start_offset token) + (leading_width token)
  93
  94 let end_offset token =
  95   let w = (width token) - 1 in
  96   let w = if w < 0 then 0 else w in
  97   (start_offset token) + w
  98
  99 let trailing_start_offset token =
 100   (leading_start_offset token) + (leading_width token) + (width token)
 101
 102 let trailing_end_offset token =
 103   let w = (full_width token) - 1 in
 104   let w = if w < 0 then 0 else w in
 105   (leading_start_offset token) + w
 106
 107 let leading_start_position token =
 108   SourceText.offset_to_position (source_text token) (leading_start_offset token)
 109
 110 let leading_end_position token =
 111   SourceText.offset_to_position (source_text token) (leading_end_offset token)
 112
 113 let start_position token =
 114   SourceText.offset_to_position (source_text token) (start_offset token)
 115
 116 let end_position token =
 117   SourceText.offset_to_position (source_text token) (end_offset token)
 118
 119 let trailing_start_position token =
 120   SourceText.offset_to_position
 121     (source_text token) (trailing_start_offset token)
 122
 123 let trailing_end_position token =
 124   SourceText.offset_to_position (source_text token) (trailing_end_offset token)
 125
 126 let leading_span token =
 127   ((leading_start_position token), (leading_end_position token))
 128
 129 let span token =
 130   ((start_position token), (end_position token))
 131
 132 let trailing_span token =
 133   ((trailing_start_position token), (trailing_end_position token))
 134
 135 let full_span token =
 136   ((leading_start_position token), (trailing_end_position token))
 137
 138 let full_text token =
 139   SourceText.sub
 140     (source_text token) (leading_start_offset token) (full_width token)
 141
 142 let leading_text token =
 143   SourceText.sub
 144     (source_text token) (leading_start_offset token) (leading_width token)
 145
 146 let trailing_text token =
 147   SourceText.sub
 148     (source_text token) ((end_offset token) + 1) (trailing_width token)
 149
 150 let text token =
 151   SourceText.sub (source_text token) (start_offset token) (width token)
 152
 153 let from_minimal source_text minimal_token offset =
 154   let kind = MinimalToken.kind minimal_token in
 155   let leading_width = MinimalToken.leading_width minimal_token in
 156   let width = MinimalToken.width minimal_token in
 157   let leading = Trivia.from_minimal_list source_text
 158     (MinimalToken.leading minimal_token) offset in
 159   let trailing = Trivia.from_minimal_list source_text
 160     (MinimalToken.trailing minimal_token) (offset + leading_width + width) in
 161   make kind source_text offset width leading trailing
 162
 163 let to_json token =
 164   let open Hh_json in
 165   let (line_number, _) = start_position token in
 166   JSON_Object [
 167     ("kind", JSON_String (TokenKind.to_string token.kind));
 168     ("text", JSON_String (text token));
 169     ("offset", int_ token.offset);
 170     ("leading_width", int_ token.leading_width);
 171     ("width", int_ token.width);
 172     ("trailing_width", int_ token.trailing_width);
 173     ("leading", JSON_Array (List.map Trivia.to_json token.leading));
 174     ("trailing", JSON_Array (List.map Trivia.to_json token.trailing));
 175     ("line_number", int_ line_number);
 176     ]