# File parsers/parse_rb.rb, line 1388
1388:     def initialize(top_level, file_name, content, options, stats)
1389:       @options = options
1390:       @stats   = stats
1391:       @size = 0
1392:       @token_listeners = nil
1393:       @input_file_name = file_name
1394:       @scanner = RubyLex.new(content)
1395:       @scanner.exception_on_syntax_error = false
1396:       @top_level = top_level
1397:       @progress = $stderr unless options.quiet
1398:     end

      # File parsers/parse_rb.rb, line 1450
1450:     def add_token_listener(obj)
1451:       @token_listeners ||= []
1452:       @token_listeners << obj
1453:     end

      # File parsers/parse_rb.rb, line 1536
1536:     def collect_first_comment
1537:       skip_tkspace
1538:       res = ''
1539:       first_line = true
1540: 
1541:       tk = get_tk
1542:       while tk.kind_of?(TkCOMMENT)
1543:         if first_line && tk.text[0,2] == "#!"
1544:           skip_tkspace
1545:           tk = get_tk
1546:         else
1547:           res << tk.text << "\n"
1548:           tk = get_tk
1549:           if tk.kind_of? TkNL
1550:             skip_tkspace(false)
1551:             tk = get_tk
1552:           end
1553:         end
1554:         first_line = false
1555:       end
1556:       unget_tk(tk)
1557:       res
1558:     end

      # File parsers/parse_rb.rb, line 1438
1438:     def error(msg)
1439:       warn msg
1440:       exit(1)
1441:     end

      # File parsers/parse_rb.rb, line 2436
2436:     def get_bool
2437:       skip_tkspace
2438:       tk = get_tk
2439:       case tk
2440:       when TkTRUE
2441:         true
2442:       when TkFALSE, TkNIL
2443:         false
2444:       else
2445:         unget_tk tk
2446:         true
2447:       end
2448:     end

      # File parsers/parse_rb.rb, line 1791
1791:     def get_class_or_module(container)
1792:       skip_tkspace
1793:       name_t = get_tk
1794: 
1795:       # class ::A -> A is in the top level
1796:       if name_t.kind_of?(TkCOLON2)
1797:         name_t = get_tk
1798:         container = @top_level
1799:       end
1800: 
1801:       skip_tkspace(false)
1802: 
1803:       while peek_tk.kind_of?(TkCOLON2)
1804:         prev_container = container
1805:         container = container.find_module_named(name_t.name)
1806:         if !container
1807: #          warn("Couldn't find module #{name_t.name}")
1808:           container = prev_container.add_module(NormalModule, name_t.name)
1809:         end
1810:         get_tk
1811:         name_t = get_tk
1812:       end
1813:       skip_tkspace(false)
1814:       return [container, name_t]
1815:     end

      # File parsers/parse_rb.rb, line 2120
2120:     def get_class_specification
2121:       tk = get_tk
2122:       return "self" if tk.kind_of?(TkSELF)
2123:         
2124:       res = ""
2125:       while tk.kind_of?(TkCOLON2) ||
2126:           tk.kind_of?(TkCOLON3)   ||
2127:           tk.kind_of?(TkCONSTANT)   
2128:         
2129:         res += tk.text
2130:         tk = get_tk
2131:       end
2132: 
2133:       unget_tk(tk)
2134:       skip_tkspace(false)
2135: 
2136:       get_tkread # empty out read buffer
2137: 
2138:       tk = get_tk
2139: 
2140:       case tk
2141:       when TkNL, TkCOMMENT, TkSEMICOLON
2142:         unget_tk(tk)
2143:         return res
2144:       end
2145: 
2146:       res += parse_call_parameters(tk)
2147:       res
2148:     end

      # File parsers/parse_rb.rb, line 2192
2192:     def get_constant
2193:       res = ""
2194:       skip_tkspace(false)
2195:       tk = get_tk
2196: 
2197:       while tk.kind_of?(TkCOLON2) ||
2198:           tk.kind_of?(TkCOLON3)   ||
2199:           tk.kind_of?(TkCONSTANT)          
2200:         
2201:         res += tk.text
2202:         tk = get_tk
2203:       end
2204: 
2205: #      if res.empty?
2206: #        warn("Unexpected token #{tk} in constant")
2207: #      end 
2208:       unget_tk(tk)
2209:       res
2210:     end

      # File parsers/parse_rb.rb, line 2214
2214:     def get_constant_with_optional_parens
2215:       skip_tkspace(false)
2216:       nest = 0
2217:       while (tk = peek_tk).kind_of?(TkLPAREN)  || tk.kind_of?(TkfLPAREN)
2218:         get_tk
2219:         skip_tkspace(true)
2220:         nest += 1
2221:       end
2222: 
2223:       name = get_constant
2224: 
2225:       while nest > 0
2226:         skip_tkspace(true)
2227:         tk = get_tk
2228:         nest -= 1 if tk.kind_of?(TkRPAREN)
2229:       end
2230:       name
2231:     end

      # File parsers/parse_rb.rb, line 2350
2350:     def get_symbol_or_name
2351:       tk = get_tk
2352:       case tk
2353:       when  TkSYMBOL
2354:         tk.text.sub(/^:/, '')
2355:       when TkId, TkOp
2356:         tk.name
2357:       when TkSTRING
2358:         tk.text
2359:       else
2360:         raise "Name or symbol expected (got #{tk})"
2361:       end
2362:     end

      # File parsers/parse_rb.rb, line 1459
1459:     def get_tk
1460:       tk = nil
1461:       if @tokens.empty?
1462:         tk = @scanner.token
1463:         @read.push @scanner.get_read
1464:         puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
1465:       else
1466:         @read.push @unget_read.shift
1467:         tk = @tokens.shift
1468:         puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
1469:       end
1470: 
1471:       if tk.kind_of?(TkSYMBEG)
1472:         set_token_position(tk.line_no, tk.char_no)
1473:         tk1 = get_tk
1474:         if tk1.kind_of?(TkId) || tk1.kind_of?(TkOp)
1475:           tk = Token(TkSYMBOL).set_text(":" + tk1.name)
1476:           # remove the identifier we just read (we're about to
1477:           # replace it with a symbol)
1478:           @token_listeners.each do |obj|
1479:             obj.pop_token
1480:           end if @token_listeners
1481:         else
1482:           warn("':' not followed by identified or operator")
1483:           tk = tk1
1484:         end
1485:       end
1486: 
1487:       # inform any listeners of our shiny new token
1488:       @token_listeners.each do |obj|
1489:         obj.add_token(tk)
1490:       end if @token_listeners
1491: 
1492:       tk
1493:     end

      # File parsers/parse_rb.rb, line 1520
1520:     def get_tkread
1521:       read = @read.join("")
1522:       @read = []
1523:       read
1524:     end

      # File parsers/parse_rb.rb, line 2299
2299:     def look_for_directives_in(context, comment)
2300: 
2301:       preprocess = SM::PreProcess.new(@input_file_name,
2302:                                       @options.rdoc_include)
2303: 
2304:       preprocess.handle(comment) do |directive, param|
2305:         case directive
2306:         when "stopdoc"
2307:           context.stop_doc
2308:           ""
2309:         when "startdoc"
2310:           context.start_doc
2311:           context.force_documentation = true
2312:           ""
2313: 
2314:         when "enddoc"
2315:           #context.done_documenting = true
2316:           #""
2317:           throw :enddoc
2318: 
2319:         when "main"
2320:           options = Options.instance
2321:           options.main_page = param
2322:           ""
2323: 
2324:         when "title"
2325:           options = Options.instance
2326:           options.title = param
2327:           ""
2328: 
2329:         when "section"
2330:           context.set_current_section(param, comment)
2331: #          comment.clear # latest version
2332:           comment.replace("") # 1.8 doesn't support #clear
2333:           break 
2334:         else
2335:           warn "Unrecognized directive '#{directive}'"
2336:           break
2337:         end
2338:       end
2339: 
2340:       remove_private_comments(comment)
2341:     end

      # File parsers/parse_rb.rb, line 2364
2364:     def parse_alias(context, single, tk, comment)
2365:       skip_tkspace
2366:       if (peek_tk.kind_of? TkLPAREN)
2367:         get_tk
2368:         skip_tkspace
2369:       end
2370:       new_name = get_symbol_or_name
2371:       @scanner.instance_eval{@lex_state = EXPR_FNAME}
2372:       skip_tkspace
2373:       if (peek_tk.kind_of? TkCOMMA)
2374:         get_tk
2375:         skip_tkspace
2376:       end
2377:       old_name = get_symbol_or_name
2378: 
2379:       al = Alias.new(get_tkread, old_name, new_name, comment)
2380:       read_documentation_modifiers(al, ATTR_MODIFIERS)
2381:       if al.document_self
2382:         context.add_alias(al)
2383:       end
2384:     end

      # File parsers/parse_rb.rb, line 2450
2450:     def parse_attr(context, single, tk, comment)
2451:       args = parse_symbol_arg(1)
2452:       if args.size > 0
2453:         name = args[0]
2454:         rw = "R"
2455:         skip_tkspace(false)
2456:         tk = get_tk
2457:         if tk.kind_of? TkCOMMA
2458:           rw = "RW" if get_bool
2459:         else
2460:           unget_tk tk
2461:         end
2462:         att = Attr.new(get_tkread, name, rw, comment)
2463:         read_documentation_modifiers(att, ATTR_MODIFIERS)
2464:         if att.document_self
2465:           context.add_attribute(att)
2466:         end
2467:       else
2468:         warn("'attr' ignored - looks like a variable")
2469:       end    
2470: 
2471:     end

      # File parsers/parse_rb.rb, line 2504
2504:     def parse_attr_accessor(context, single, tk, comment)
2505:       args = parse_symbol_arg
2506:       read = get_tkread
2507:       rw = "?"
2508: 
2509:       # If nodoc is given, don't document any of them
2510: 
2511:       tmp = CodeObject.new
2512:       read_documentation_modifiers(tmp, ATTR_MODIFIERS)
2513:       return unless tmp.document_self
2514: 
2515:       case tk.name
2516:       when "attr_reader"   then rw = "R"
2517:       when "attr_writer"   then rw = "W"
2518:       when "attr_accessor" then rw = "RW"
2519:       else
2520:         rw = @options.extra_accessor_flags[tk.name]
2521:       end
2522:       
2523:       for name in args
2524:         att = Attr.new(get_tkread, name, rw, comment)
2525:         context.add_attribute(att)
2526:       end    
2527:     end

      # File parsers/parse_rb.rb, line 2150
2150:     def parse_call_parameters(tk)
2151: 
2152:       end_token = case tk
2153:                   when TkLPAREN, TkfLPAREN
2154:                     TkRPAREN
2155:                   when TkRPAREN
2156:                     return ""
2157:                   else
2158:                     TkNL
2159:                   end
2160:       nest = 0
2161: 
2162:       loop do
2163:         puts("Call param: #{tk}, #{@scanner.continue} " +
2164:           "#{@scanner.lex_state} #{nest}") if $DEBUG
2165:         case tk
2166:         when TkSEMICOLON
2167:           break
2168:         when TkLPAREN, TkfLPAREN
2169:           nest += 1
2170:         when end_token
2171:           if end_token == TkRPAREN
2172:             nest -= 1
2173:             break if @scanner.lex_state == EXPR_END and nest <= 0
2174:           else
2175:             break unless @scanner.continue
2176:           end
2177:         when TkCOMMENT
2178:           unget_tk(tk)
2179:           break
2180:         end
2181:         tk = get_tk
2182:       end
2183:       res = get_tkread.tr("\n", " ").strip
2184:       res = "" if res == ";"
2185:       res
2186:     end

      # File parsers/parse_rb.rb, line 1723
1723:     def parse_class(container, single, tk, comment, &block)
1724:       progress("c")
1725: 
1726:       @stats.num_classes += 1
1727: 
1728:       container, name_t = get_class_or_module(container)
1729: 
1730:       case name_t
1731:       when TkCONSTANT
1732:         name = name_t.name
1733:         superclass = "Object"
1734: 
1735:         if peek_tk.kind_of?(TkLT)
1736:           get_tk
1737:           skip_tkspace(true)
1738:           superclass = get_class_specification
1739:           superclass = "<unknown>" if superclass.empty?
1740:         end
1741: 
1742:         if single == SINGLE
1743:           cls_type = SingleClass
1744:         else
1745:           cls_type = NormalClass
1746:         end
1747: 
1748:         cls = container.add_class(cls_type, name, superclass)
1749:         read_documentation_modifiers(cls, CLASS_MODIFIERS)
1750:         cls.record_location(@top_level)
1751:         parse_statements(cls)
1752:         cls.comment = comment
1753: 
1754:       when TkLSHFT
1755:         case name = get_class_specification
1756:         when "self", container.name
1757:           parse_statements(container, SINGLE, &block)
1758:         else
1759:           other = TopLevel.find_class_named(name)
1760:           unless other
1761: #            other = @top_level.add_class(NormalClass, name, nil)
1762: #            other.record_location(@top_level)
1763: #            other.comment = comment
1764:             other = NormalClass.new("Dummy", nil)
1765:           end
1766:           read_documentation_modifiers(other, CLASS_MODIFIERS)
1767:           parse_statements(other, SINGLE, &block)
1768:         end
1769: 
1770:       else
1771:         warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}")
1772:       end
1773:     end

      # File parsers/parse_rb.rb, line 1817
1817:     def parse_constant(container, single, tk, comment)
1818:       name = tk.name
1819:       skip_tkspace(false)
1820:       eq_tk = get_tk
1821: 
1822:       unless eq_tk.kind_of?(TkASSIGN)
1823:         unget_tk(eq_tk)
1824:         return
1825:       end
1826: 
1827: 
1828:       nest = 0
1829:       get_tkread
1830: 
1831:       tk = get_tk
1832:       if tk.kind_of? TkGT
1833:         unget_tk(tk)
1834:         unget_tk(eq_tk)
1835:         return
1836:       end
1837: 
1838:       loop do
1839:         puts("Param: #{tk}, #{@scanner.continue} " +
1840:           "#{@scanner.lex_state} #{nest}")  if $DEBUG
1841: 
1842:         case tk
1843:         when TkSEMICOLON
1844:           break
1845:         when TkLPAREN, TkfLPAREN
1846:           nest += 1
1847:         when TkRPAREN
1848:           nest -= 1
1849:         when TkCOMMENT
1850:           if nest <= 0 && @scanner.lex_state == EXPR_END
1851:             unget_tk(tk)
1852:             break
1853:           end
1854:         when TkNL
1855:           if (@scanner.lex_state == EXPR_END and nest <= 0) || !@scanner.continue
1856:             unget_tk(tk)
1857:             break
1858:           end
1859:         end
1860:         tk = get_tk
1861:       end
1862: 
1863:       res = get_tkread.tr("\n", " ").strip
1864:       res = "" if res == ";"
1865:       con = Constant.new(name, res, comment)
1866:       read_documentation_modifiers(con, CONSTANT_MODIFIERS)
1867:       if con.document_self
1868:         container.add_constant(con)
1869:       end
1870:     end

      # File parsers/parse_rb.rb, line 2424
2424:   def parse_include(context, comment)
2425:     loop do
2426:       skip_tkspace_comment
2427:       name = get_constant_with_optional_parens
2428:       unless name.empty?
2429:         context.add_include(Include.new(name, comment))
2430:       end
2431:       return unless peek_tk.kind_of?(TkCOMMA)
2432:       get_tk
2433:     end
2434:   end

      # File parsers/parse_rb.rb, line 1872
1872:     def parse_method(container, single, tk, comment)
1873:       progress(".")
1874:       @stats.num_methods += 1
1875:       line_no = tk.line_no
1876:       column  = tk.char_no
1877:       
1878:       start_collecting_tokens
1879:       add_token(tk)
1880:       add_token_listener(self)
1881:       
1882:       @scanner.instance_eval{@lex_state = EXPR_FNAME}
1883:       skip_tkspace(false)
1884:       name_t = get_tk
1885:       back_tk = skip_tkspace
1886:       meth = nil
1887:       added_container = false
1888: 
1889:       dot = get_tk
1890:       if dot.kind_of?(TkDOT) or dot.kind_of?(TkCOLON2)
1891:         @scanner.instance_eval{@lex_state = EXPR_FNAME}
1892:         skip_tkspace
1893:         name_t2 = get_tk
1894:         case name_t
1895:         when TkSELF
1896:           name = name_t2.name
1897:         when TkCONSTANT
1898:           name = name_t2.name
1899:           prev_container = container
1900:           container = container.find_module_named(name_t.name)
1901:           if !container
1902:             added_container = true
1903:             obj = name_t.name.split("::").inject(Object) do |state, item|
1904:               state.const_get(item)
1905:             end rescue nil
1906: 
1907:             type = obj.class == Class ? NormalClass : NormalModule
1908:             if not [Class, Module].include?(obj.class)
1909:               warn("Couldn't find #{name_t.name}. Assuming it's a module")
1910:             end
1911: 
1912:             if type == NormalClass then
1913:               container = prev_container.add_class(type, name_t.name, obj.superclass.name)
1914:             else
1915:               container = prev_container.add_module(type, name_t.name)
1916:             end
1917:           end
1918:         else
1919:           # warn("Unexpected token '#{name_t2.inspect}'")
1920:           # break
1921:           skip_method(container)
1922:           return
1923:         end
1924:         meth =  AnyMethod.new(get_tkread, name)
1925:         meth.singleton = true
1926:       else
1927:         unget_tk dot
1928:         back_tk.reverse_each do
1929:           |tk|
1930:           unget_tk tk
1931:         end
1932:         name = name_t.name
1933: 
1934:         meth =  AnyMethod.new(get_tkread, name)
1935:         meth.singleton = (single == SINGLE)
1936:       end
1937: 
1938:       remove_token_listener(self)
1939: 
1940:       meth.start_collecting_tokens
1941:       indent = TkSPACE.new(1,1)
1942:       indent.set_text(" " * column)
1943: 
1944:       meth.add_tokens([TkCOMMENT.new(line_no,
1945:                                      1,
1946:                                      "# File #{@top_level.file_absolute_name}, line #{line_no}"),
1947:                         NEWLINE_TOKEN,
1948:                         indent])
1949: 
1950:       meth.add_tokens(@token_stream)
1951: 
1952:       add_token_listener(meth)
1953: 
1954:       @scanner.instance_eval{@continue = false}
1955:       parse_method_parameters(meth)
1956: 
1957:       if meth.document_self
1958:         container.add_method(meth)
1959:       elsif added_container
1960:         container.document_self = false
1961:       end
1962: 
1963:       # Having now read the method parameters and documentation modifiers, we
1964:       # now know whether we have to rename #initialize to ::new
1965: 
1966:       if name == "initialize" && !meth.singleton
1967:         if meth.dont_rename_initialize
1968:           meth.visibility = :protected
1969:         else
1970:           meth.singleton = true
1971:           meth.name = "new"
1972:           meth.visibility = :public
1973:         end
1974:       end
1975:       
1976:       parse_statements(container, single, meth)
1977:       
1978:       remove_token_listener(meth)
1979: 
1980:       # Look for a 'call-seq' in the comment, and override the
1981:       # normal parameter stuff
1982: 
1983:       if comment.sub!(/:?call-seq:(.*?)^\s*\#?\s*$/m, '')
1984:         seq = $1
1985:         seq.gsub!(/^\s*\#\s*/, '')
1986:         meth.call_seq = seq
1987:       end
1988:       
1989:       meth.comment = comment
1990: 
1991:     end

      # File parsers/parse_rb.rb, line 2016
2016:     def parse_method_or_yield_parameters(method=nil, modifiers=METHOD_MODIFIERS)
2017:       skip_tkspace(false)
2018:       tk = get_tk
2019: 
2020:       # Little hack going on here. In the statement
2021:       #  f = 2*(1+yield)
2022:       # We see the RPAREN as the next token, so we need
2023:       # to exit early. This still won't catch all cases
2024:       # (such as "a = yield + 1"
2025:       end_token = case tk
2026:                   when TkLPAREN, TkfLPAREN
2027:                     TkRPAREN
2028:                   when TkRPAREN
2029:                     return ""
2030:                   else
2031:                     TkNL
2032:                   end
2033:       nest = 0
2034: 
2035:       loop do
2036:         puts("Param: #{tk.inspect}, #{@scanner.continue} " +
2037:           "#{@scanner.lex_state} #{nest}")  if $DEBUG
2038:         case tk
2039:         when TkSEMICOLON
2040:           break
2041:         when TkLBRACE
2042:           nest += 1
2043:         when TkRBRACE
2044:           # we might have a.each {|i| yield i }
2045:           unget_tk(tk) if nest.zero?
2046:           nest -= 1
2047:           break if nest <= 0
2048:         when TkLPAREN, TkfLPAREN
2049:           nest += 1
2050:         when end_token
2051:           if end_token == TkRPAREN
2052:             nest -= 1
2053:             break if @scanner.lex_state == EXPR_END and nest <= 0
2054:           else
2055:             break unless @scanner.continue
2056:           end
2057:         when method && method.block_params.nil? && TkCOMMENT
2058:           unget_tk(tk)
2059:           read_documentation_modifiers(method, modifiers)
2060:         end
2061:         tk = get_tk
2062:       end
2063:       res = get_tkread.tr("\n", " ").strip
2064:       res = "" if res == ";"
2065:       res
2066:     end

      # File parsers/parse_rb.rb, line 2006
2006:     def parse_method_parameters(method)
2007:       res = parse_method_or_yield_parameters(method)
2008:       res = "(" + res + ")" unless res[0] == ?(
2009:       method.params = res unless method.params
2010:       if method.block_params.nil?
2011:           skip_tkspace(false)
2012:           read_documentation_modifiers(method, METHOD_MODIFIERS)
2013:       end
2014:     end

      # File parsers/parse_rb.rb, line 1775
1775:     def parse_module(container, single, tk, comment)
1776:       progress("m")
1777:       @stats.num_modules += 1
1778:       container, name_t  = get_class_or_module(container)
1779: #      skip_tkspace
1780:       name = name_t.name
1781:       mod = container.add_module(NormalModule, name)
1782:       mod.record_location(@top_level)
1783:       read_documentation_modifiers(mod, CLASS_MODIFIERS)
1784:       parse_statements(mod)
1785:       mod.comment = comment
1786:     end

      # File parsers/parse_rb.rb, line 2398
2398:   def parse_require(context, comment)
2399:     skip_tkspace_comment
2400:     tk = get_tk
2401:     if tk.kind_of? TkLPAREN
2402:       skip_tkspace_comment
2403:       tk = get_tk
2404:     end
2405: 
2406:     name = nil
2407:     case tk
2408:     when TkSTRING
2409:       name = tk.text
2410: #    when TkCONSTANT, TkIDENTIFIER, TkIVAR, TkGVAR
2411: #      name = tk.name
2412:     when TkDSTRING
2413:       warn "Skipping require of dynamic string: #{tk.text}"
2414:  #   else
2415:  #     warn "'require' used as variable"
2416:     end
2417:     if name
2418:       context.add_require(Require.new(name, comment))
2419:     else
2420:       unget_tk(tk)
2421:     end
2422:   end

      # File parsers/parse_rb.rb, line 1567
1567:     def parse_statements(container, single=NORMAL, current_method=nil, comment='')
1568:       nest = 1
1569:       save_visibility = container.visibility
1570:       
1571: #      if container.kind_of?(TopLevel)
1572: #      else
1573: #        comment = ''
1574: #      end
1575: 
1576:       non_comment_seen = true
1577:       
1578:       while tk = get_tk
1579:         
1580:         keep_comment = false
1581:         
1582:         non_comment_seen = true unless tk.kind_of?(TkCOMMENT)
1583:         
1584:         case tk
1585: 
1586:         when TkNL
1587:           skip_tkspace(true)   # Skip blanks and newlines
1588:           tk = get_tk
1589:           if tk.kind_of?(TkCOMMENT)
1590:             if non_comment_seen
1591:               comment = ''
1592:               non_comment_seen = false
1593:             end
1594:             while tk.kind_of?(TkCOMMENT)
1595:               comment << tk.text << "\n"
1596:               tk = get_tk          # this is the newline 
1597:               skip_tkspace(false)  # leading spaces
1598:               tk = get_tk
1599:             end
1600:             unless comment.empty?
1601:               look_for_directives_in(container, comment) 
1602:               if container.done_documenting
1603:                 container.ongoing_visibility = save_visibility
1604: #                return
1605:               end
1606:             end
1607:             keep_comment = true
1608:           else
1609:             non_comment_seen = true
1610:           end
1611:           unget_tk(tk)
1612:           keep_comment = true
1613: 
1614: 
1615:         when TkCLASS
1616:           if container.document_children
1617:             parse_class(container, single, tk, comment)
1618:           else
1619:             nest += 1
1620:           end
1621: 
1622:         when TkMODULE
1623:           if container.document_children
1624:             parse_module(container, single, tk, comment)
1625:           else
1626:             nest += 1
1627:           end
1628: 
1629:         when TkDEF
1630:           if container.document_self
1631:             parse_method(container, single, tk, comment)
1632:           else
1633:             nest += 1
1634:           end
1635: 
1636:         when TkCONSTANT
1637:           if container.document_self
1638:             parse_constant(container, single, tk, comment)
1639:           end
1640: 
1641:         when TkALIAS
1642:           if container.document_self
1643:             parse_alias(container, single, tk, comment)
1644:           end
1645: 
1646:         when TkYIELD
1647:           if current_method.nil?
1648:             warn("Warning: yield outside of method") if container.document_self
1649:           else
1650:             parse_yield(container, single, tk, current_method)
1651:           end
1652: 
1653:           # Until and While can have a 'do', which shouldn't increas
1654:           # the nesting. We can't solve the general case, but we can
1655:           # handle most occurrences by ignoring a do at the end of a line
1656: 
1657:         when  TkUNTIL, TkWHILE
1658:           nest += 1
1659:           puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
1660:             "line #{tk.line_no}" if $DEBUG
1661:           skip_optional_do_after_expression
1662: 
1663:           # 'for' is trickier
1664:         when TkFOR
1665:           nest += 1
1666:           puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
1667:             "line #{tk.line_no}" if $DEBUG
1668:           skip_for_variable
1669:           skip_optional_do_after_expression
1670: 
1671:         when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN
1672:           nest += 1
1673:           puts "Found #{tk.class} in #{container.name}, nest = #{nest}, " +
1674:             "line #{tk.line_no}" if $DEBUG
1675: 
1676:         when TkIDENTIFIER
1677:           if nest == 1 and current_method.nil?
1678:             case tk.name
1679:             when "private", "protected", "public",
1680:                  "private_class_method", "public_class_method"
1681:               parse_visibility(container, single, tk)
1682:               keep_comment = true
1683:             when "attr"
1684:               parse_attr(container, single, tk, comment)
1685:             when /^attr_(reader|writer|accessor)$/, @options.extra_accessors
1686:               parse_attr_accessor(container, single, tk, comment)
1687:             when "alias_method"
1688:               if container.document_self
1689:                 parse_alias(container, single, tk, comment)
1690:               end
1691:             end
1692:           end
1693:           
1694:           case tk.name
1695:           when "require"
1696:             parse_require(container, comment)
1697:           when "include"
1698:             parse_include(container, comment)
1699:           end
1700: 
1701: 
1702:         when TkEND
1703:           nest -= 1
1704:           puts "Found 'end' in #{container.name}, nest = #{nest}, line #{tk.line_no}" if $DEBUG
1705:           puts "Method = #{current_method.name}" if $DEBUG and current_method
1706:           if nest == 0
1707:             read_documentation_modifiers(container, CLASS_MODIFIERS)
1708:             container.ongoing_visibility = save_visibility
1709:             return
1710:           end
1711: 
1712:         end
1713: 
1714:         comment = '' unless keep_comment
1715:         begin
1716:           get_tkread
1717:           skip_tkspace(false)
1718:         end while peek_tk == TkNL
1719: 
1720:       end
1721:     end

      # File parsers/parse_rb.rb, line 2537
2537:     def parse_symbol_arg(no = nil)
2538: 
2539:       args = []
2540:       skip_tkspace_comment
2541:       case tk = get_tk
2542:       when TkLPAREN
2543:         loop do
2544:           skip_tkspace_comment
2545:           if tk1 = parse_symbol_in_arg
2546:             args.push tk1
2547:             break if no and args.size >= no
2548:           end
2549:           
2550:           skip_tkspace_comment
2551:           case tk2 = get_tk
2552:           when TkRPAREN
2553:             break
2554:           when TkCOMMA
2555:           else
2556:             warn("unexpected token: '#{tk2.inspect}'") if $DEBBUG
2557:             break
2558:           end
2559:         end
2560:       else
2561:         unget_tk tk
2562:         if tk = parse_symbol_in_arg
2563:           args.push tk
2564:           return args if no and args.size >= no
2565:         end
2566: 
2567:         loop do
2568: #         skip_tkspace_comment(false)
2569:           skip_tkspace(false)
2570: 
2571:           tk1 = get_tk
2572:           unless tk1.kind_of?(TkCOMMA) 
2573:             unget_tk tk1
2574:             break
2575:           end
2576:           
2577:           skip_tkspace_comment
2578:           if tk = parse_symbol_in_arg
2579:             args.push tk
2580:             break if no and args.size >= no
2581:           end
2582:         end
2583:       end
2584:       args
2585:     end

      # File parsers/parse_rb.rb, line 2587
2587:     def parse_symbol_in_arg
2588:       case tk = get_tk
2589:       when TkSYMBOL
2590:         tk.text.sub(/^:/, '')
2591:       when TkSTRING
2592:         eval @read[-1]
2593:       else
2594:         warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG
2595:         nil
2596:       end
2597:     end

      # File parsers/parse_rb.rb, line 1560
1560:     def parse_toplevel_statements(container)
1561:       comment = collect_first_comment
1562:       look_for_directives_in(container, comment)
1563:       container.comment = comment unless comment.empty?
1564:       parse_statements(container, NORMAL, nil, comment)
1565:     end

      # File parsers/parse_rb.rb, line 2473
2473:     def parse_visibility(container, single, tk)
2474:       singleton = (single == SINGLE)
2475:       vis = case tk.name
2476:             when "private"   then :private
2477:             when "protected" then :protected
2478:             when "public"    then :public
2479:             when "private_class_method"
2480:               singleton = true
2481:               :private
2482:             when "public_class_method"
2483:               singleton = true
2484:               :public
2485:             else raise "Invalid visibility: #{tk.name}"
2486:             end
2487:             
2488:       skip_tkspace_comment(false)
2489:       case peek_tk
2490:         # Ryan Davis suggested the extension to ignore modifiers, because he
2491:         # often writes
2492:         #
2493:         #   protected unless $TESTING
2494:         #
2495:       when TkNL, TkUNLESS_MOD, TkIF_MOD
2496: #        error("Missing argument") if singleton        
2497:         container.ongoing_visibility = vis
2498:       else
2499:         args = parse_symbol_arg
2500:         container.set_visibility_for(args, vis, singleton)
2501:       end
2502:     end

      # File parsers/parse_rb.rb, line 2390
2390:   def parse_yield(context, single, tk, method)
2391:     if method.block_params.nil?
2392:       get_tkread
2393:       @scanner.instance_eval{@continue = false}
2394:       method.block_params = parse_yield_parameters
2395:     end
2396:   end

      # File parsers/parse_rb.rb, line 2386
2386:     def parse_yield_parameters
2387:       parse_method_or_yield_parameters
2388:     end

      # File parsers/parse_rb.rb, line 1526
1526:     def peek_read
1527:       @read.join('')
1528:     end

      # File parsers/parse_rb.rb, line 1495
1495:     def peek_tk
1496:       unget_tk(tk = get_tk)
1497:       tk
1498:     end

      # File parsers/parse_rb.rb, line 1443
1443:     def progress(char)
1444:       unless @options.quiet
1445:         @progress.print(char)
1446:         @progress.flush
1447:       end
1448:     end

      # File parsers/parse_rb.rb, line 2244
2244:     def read_directive(allowed)
2245:       tk = get_tk
2246:       puts "directive: #{tk.inspect}" if $DEBUG
2247:       result = nil
2248:       if tk.kind_of?(TkCOMMENT) 
2249:         if tk.text =~ /\s*:?(\w+):\s*(.*)/
2250:           directive = $1.downcase
2251:           if allowed.include?(directive)
2252:             result = [directive, $2]
2253:           end
2254:         end
2255:       else
2256:         unget_tk(tk)
2257:       end
2258:       result
2259:     end

      # File parsers/parse_rb.rb, line 2262
2262:     def read_documentation_modifiers(context, allow)
2263:       dir = read_directive(allow)
2264: 
2265:       case dir[0]
2266: 
2267:       when "notnew", "not_new", "not-new"
2268:         context.dont_rename_initialize = true
2269: 
2270:       when "nodoc"
2271:         context.document_self = false
2272:         if dir[1].downcase == "all"
2273:           context.document_children = false
2274:         end
2275: 
2276:       when "doc"
2277:         context.document_self = true
2278:         context.force_documentation = true
2279: 
2280:       when "yield", "yields"
2281:         unless context.params.nil?
2282:           context.params.sub!(/(,|)\s*&\w+/,'') # remove parameter &proc
2283:         end
2284:         context.block_params = dir[1]
2285: 
2286:       when "arg", "args"
2287:         context.params = dir[1]
2288:       end if dir
2289:     end

      # File parsers/parse_rb.rb, line 2343
2343:     def remove_private_comments(comment)
2344:       comment.gsub!(/^#--.*?^#\+\+/m, '')
2345:       comment.sub!(/^#--.*/m, '')
2346:     end

      # File parsers/parse_rb.rb, line 1455
1455:     def remove_token_listener(obj)
1456:       @token_listeners.delete(obj)
1457:     end

      # File parsers/parse_rb.rb, line 1400
1400:     def scan
1401:       @tokens = []
1402:       @unget_read = []
1403:       @read = []
1404:       catch(:eof) do
1405:         catch(:enddoc) do
1406:           begin
1407:             parse_toplevel_statements(@top_level)
1408:           rescue Exception => e
1409:             $stderr.puts "\n\n"
1410:             $stderr.puts "RDoc failure in #@input_file_name at or around " +
1411:                          "line #{@scanner.line_no} column #{@scanner.char_no}"
1412:             $stderr.puts 
1413:             $stderr.puts "Before reporting this, could you check that the file"
1414:             $stderr.puts "you're documenting compiles cleanly--RDoc is not a"
1415:             $stderr.puts "full Ruby parser, and gets confused easily if fed"
1416:             $stderr.puts "invalid programs."
1417:             $stderr.puts
1418:             $stderr.puts "The internal error was:\n\n"
1419:             
1420:             e.set_backtrace(e.backtrace[0,4])
1421:             raise
1422:           end
1423:         end
1424:       end
1425:       @top_level
1426:     end

      # File parsers/parse_rb.rb, line 2069
2069:     def skip_for_variable
2070:       skip_tkspace(false)
2071:       tk = get_tk
2072:       skip_tkspace(false)
2073:       tk = get_tk
2074:       unget_tk(tk) unless tk.kind_of?(TkIN)
2075:     end

      # File parsers/parse_rb.rb, line 1993
1993:     def skip_method(container)
1994:       meth =  AnyMethod.new("", "anon")
1995:       parse_method_parameters(meth)
1996:       parse_statements(container, false, meth)
1997:     end

      # File parsers/parse_rb.rb, line 2078
2078:     def skip_optional_do_after_expression
2079:       skip_tkspace(false)
2080:       tk = get_tk
2081:       case tk
2082:       when TkLPAREN, TkfLPAREN
2083:         end_token = TkRPAREN
2084:       else
2085:         end_token = TkNL
2086:       end
2087: 
2088:       nest = 0
2089:       @scanner.instance_eval{@continue = false}
2090: 
2091:       loop do
2092:         puts("\nWhile: #{tk}, #{@scanner.continue} " +
2093:           "#{@scanner.lex_state} #{nest}") if $DEBUG
2094:         case tk
2095:         when TkSEMICOLON
2096:           break
2097:         when TkLPAREN, TkfLPAREN
2098:           nest += 1
2099:         when TkDO
2100:           break if nest.zero?
2101:         when end_token
2102:           if end_token == TkRPAREN
2103:             nest -= 1
2104:             break if @scanner.lex_state == EXPR_END and nest.zero?
2105:           else
2106:             break unless @scanner.continue
2107:           end
2108:         end
2109:         tk = get_tk
2110:       end
2111:       skip_tkspace(false)
2112:       if peek_tk.kind_of? TkDO
2113:         get_tk
2114:       end
2115:     end

      # File parsers/parse_rb.rb, line 1510
1510:     def skip_tkspace(skip_nl = true)
1511:       tokens = []
1512:       while ((tk = get_tk).kind_of?(TkSPACE) ||
1513:              (skip_nl && tk.kind_of?(TkNL)))
1514:         tokens.push tk
1515:       end
1516:       unget_tk(tk)
1517:       tokens
1518:     end

      # File parsers/parse_rb.rb, line 2529
2529:     def skip_tkspace_comment(skip_nl = true)
2530:       loop do
2531:         skip_tkspace(skip_nl)
2532:         return unless peek_tk.kind_of? TkCOMMENT
2533:         get_tk
2534:       end
2535:     end

      # File parsers/parse_rb.rb, line 1500
1500:     def unget_tk(tk)
1501:       @tokens.unshift tk
1502:       @unget_read.unshift @read.pop
1503: 
1504:       # Remove this token from any listeners
1505:       @token_listeners.each do |obj|
1506:         obj.pop_token
1507:       end if @token_listeners
1508:     end

      # File parsers/parse_rb.rb, line 1430
1430:     def warn(msg)
1431:       prefix = "\n" + @input_file_name + ":"
1432:       if @scanner
1433:         prefix << "#{@scanner.line_no}:#{@scanner.char_no}: "
1434:       end
1435:       $stderr.puts prefix + msg
1436:     end