Class: GoodData::Bricks::BaseDownloader

Inherits:
Object
  • Object
show all
Defined in:
lib/gooddata/bricks/base_downloader.rb

Instance Method Summary collapse

Constructor Details

#initialize(params) ⇒ BaseDownloader

Returns a new instance of BaseDownloader.



13
14
15
16
 
# File 'lib/gooddata/bricks/base_downloader.rb', line 13

def initialize(params)
  @params = params
  @logger = @params['GDC_LOGGER']
end

Instance Method Details

#backup(meta) ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/gooddata/bricks/base_downloader.rb', line 27

def backup(meta)
  @logger.info 'would send a backup list of files to backup' if @logger

  files = meta['objects'].map { |_k, o| o }.reduce([]) do |a, e|
    a + e['filenames']
  end

  bucket_name = @params['s3_backup_bucket_name']

  s3 = Aws::S3::Resource.new(
    :access_key_id => @params['aws_access_key_id'],
    :secret_access_key => @params['aws_secret_access_key']
  )

  bucket = s3.bucket(bucket_name)
  bucket = s3.create_bucket(bucket_name) unless bucket.exists?

  files.each do |file|
    file_path = Pathname.new(file)
    target_path = Pathname.new(@params['s3_backup_path'] || '') + file_path.basename
    obj = bucket.objects(target_path)
    obj.upload_file(file_path)
    @logger.info "Backed up file #{file_path} to s3 #{target_path}" if @logger
  end

  meta
end

#download ⇒ Object 



22
23
24
25
 
# File 'lib/gooddata/bricks/base_downloader.rb', line 22

def download
  @logger.info 'would download data' if @logger
  []
end

#post_process(meta) ⇒ Object 



55
56
57
58
 
# File 'lib/gooddata/bricks/base_downloader.rb', line 55

def post_process(meta)
  @logger.info 'Maybe some postprocessing' if @logger
  meta
end

#pre_process(meta) ⇒ Object 



18
19
20
 
# File 'lib/gooddata/bricks/base_downloader.rb', line 18

def pre_process(meta)
  meta
end

#run ⇒ Object

Run downloader



61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/gooddata/bricks/base_downloader.rb', line 61

def run
  downloaded_data = download
  downloaded_data = pre_process(downloaded_data)
  backup(downloaded_data) unless @params['skip_backup']

  downloaded_data = post_process(downloaded_data)

  # save the state - whatever is in the return value of #download in the :persist key .. to project metadata
  if downloaded_data[:persist]
    accumulated_state = downloaded_data[:persist].reduce([]) do |memo, item|
      item.key?(:state) ? memo.concat(item[:state]) : memo
    end
    accumulated_state.each do |item|
      key = item[:key]
      val = item[:value]

      @logger.info "Saving metadata #{key} => #{val}" if @logger
      GoodData::ProjectMetadata[key] = val
    end
  end

  post_process(downloaded_data)
end